ARROW-5757: [Python] Remove Python 2.7 support

Part of the changes were done using [pyupgrade](https://github.com/asottile/pyupgrade).

.github/workflows/python.yml

@@ -72,7 +72,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: [2.7, 3.6, 3.8]
+        python: [3.6, 3.8]
     env:
       PYTHON: ${{ matrix.python }}
       PANDAS: latest

ci/docker/conda-python.dockerfile

@@ -24,7 +24,6 @@ ARG python=3.6
 COPY ci/conda_env_python.yml /arrow/ci/
 RUN conda install -q \
         --file arrow/ci/conda_env_python.yml \
-        $([ "$python" == "2.7" ] && echo "futures enum34") \
         $([ "$python" == "3.6" -o "$python" == "3.7" ] && echo "pickle5") \
         python=${python} \
         nomkl && \

cpp/src/arrow/python/common.h

@@ -134,8 +134,6 @@ auto SafeCallIntoPython(Function&& func) -> decltype(func()) {
   return maybe_status;
 }
 
-#define PYARROW_IS_PY2 PY_MAJOR_VERSION <= 2
-
 // A RAII primitive that DECREFs the underlying PyObject* when it
 // goes out of scope.
 class ARROW_PYTHON_EXPORT OwnedRef {
@@ -211,7 +209,6 @@ struct PyBytesView {
   }
 
   Status FromUnicode(PyObject* obj) {
-#if PY_MAJOR_VERSION >= 3
     Py_ssize_t size;
     // The utf-8 representation is cached on the unicode object
     const char* data = PyUnicode_AsUTF8AndSize(obj, &size);
@@ -219,13 +216,6 @@ struct PyBytesView {
     this->bytes = data;
     this->size = size;
     this->ref.reset();
-#else
-    PyObject* converted = PyUnicode_AsUTF8String(obj);
-    RETURN_IF_PYERROR();
-    this->bytes = PyBytes_AS_STRING(converted);
-    this->size = PyBytes_GET_SIZE(converted);
-    this->ref.reset(converted);
-#endif
     return Status::OK();
   }
 

cpp/src/arrow/python/config.h

@@ -23,10 +23,6 @@
 #include "arrow/python/numpy_interop.h"
 #include "arrow/python/visibility.h"
 
-#if PY_MAJOR_VERSION >= 3
-#define PyString_Check PyUnicode_Check
-#endif
-
 namespace arrow {
 namespace py {
 

cpp/src/arrow/python/datetime.h

@@ -121,24 +121,15 @@ inline TimePoint PyDateTime_to_TimePoint(PyDateTime_DateTime* pydatetime) {
 ARROW_PYTHON_EXPORT
 inline int64_t PyDelta_to_s(PyDateTime_Delta* pytimedelta) {
   int64_t total_seconds = 0;
-#if PY_VERSION_HEX >= 0x03000000
   total_seconds += PyDateTime_DELTA_GET_SECONDS(pytimedelta);
   total_seconds += PyDateTime_DELTA_GET_DAYS(pytimedelta) * 24 * 3600;
-#else
-  total_seconds += pytimedelta->seconds;
-  total_seconds += pytimedelta->days * 24 * 3600;
-#endif
   return total_seconds;
 }
 
 ARROW_PYTHON_EXPORT
 inline int64_t PyDelta_to_ms(PyDateTime_Delta* pytimedelta) {
   int64_t total_ms = PyDelta_to_s(pytimedelta) * 1000;
-#if PY_VERSION_HEX >= 0x03000000
   total_ms += PyDateTime_DELTA_GET_MICROSECONDS(pytimedelta) / 1000;
-#else
-  total_ms += pytimedelta->microseconds / 1000;
-#endif
   return total_ms;
 }
 
@@ -146,11 +137,7 @@ ARROW_PYTHON_EXPORT
 inline int64_t PyDelta_to_us(PyDateTime_Delta* pytimedelta) {
   int64_t total_us = 0;
   total_us += PyDelta_to_s(pytimedelta) * 1000 * 1000;
-#if PY_VERSION_HEX >= 0x03000000
   total_us += PyDateTime_DELTA_GET_MICROSECONDS(pytimedelta);
-#else
-  total_us += pytimedelta->microseconds;
-#endif
   return total_us;
 }
 

cpp/src/arrow/python/deserialize.cc

@@ -123,12 +123,6 @@ Status GetValue(PyObject* context, const Array& arr, int64_t index, int8_t type,
       return Status::OK();
     case PythonType::PY2INT:
     case PythonType::INT: {
-#if PY_MAJOR_VERSION < 3
-      if (type == PythonType::PY2INT) {
-        *result = PyInt_FromSsize_t(checked_cast<const Int64Array&>(arr).Value(index));
-        return Status::OK();
-      }
-#endif
       *result = PyLong_FromSsize_t(checked_cast<const Int64Array&>(arr).Value(index));
       return Status::OK();
     }

cpp/src/arrow/python/helpers.cc

@@ -96,23 +96,15 @@ std::string PyBytes_AsStdString(PyObject* obj) {
 
 Status PyUnicode_AsStdString(PyObject* obj, std::string* out) {
   DCHECK(PyUnicode_Check(obj));
-#if PY_MAJOR_VERSION >= 3
   Py_ssize_t size;
   // The utf-8 representation is cached on the unicode object
   const char* data = PyUnicode_AsUTF8AndSize(obj, &size);
   RETURN_IF_PYERROR();
   *out = std::string(data, size);
   return Status::OK();
-#else
-  OwnedRef bytes_ref(PyUnicode_AsUTF8String(obj));
-  RETURN_IF_PYERROR();
-  *out = PyBytes_AsStdString(bytes_ref.obj());
-  return Status::OK();
-#endif
 }
 
 std::string PyObject_StdStringRepr(PyObject* obj) {
-#if PY_MAJOR_VERSION >= 3
   OwnedRef unicode_ref(PyObject_Repr(obj));
   OwnedRef bytes_ref;
 
@@ -120,9 +112,6 @@ std::string PyObject_StdStringRepr(PyObject* obj) {
     bytes_ref.reset(
         PyUnicode_AsEncodedString(unicode_ref.obj(), "utf8", "backslashreplace"));
   }
-#else
-  OwnedRef bytes_ref(PyObject_Repr(obj));
-#endif
   if (!bytes_ref) {
     PyErr_Clear();
     std::stringstream ss;
@@ -135,12 +124,7 @@ std::string PyObject_StdStringRepr(PyObject* obj) {
 Status PyObject_StdStringStr(PyObject* obj, std::string* out) {
   OwnedRef string_ref(PyObject_Str(obj));
   RETURN_IF_PYERROR();
-#if PY_MAJOR_VERSION >= 3
   return PyUnicode_AsStdString(string_ref.obj(), out);
-#else
-  *out = PyBytes_AsStdString(string_ref.obj());
-  return Status::OK();
-#endif
 }
 
 Status ImportModule(const std::string& module_name, OwnedRef* ref) {
@@ -259,18 +243,10 @@ template Status CIntFromPython(PyObject*, uint64_t*, const std::string&);
 
 inline bool MayHaveNaN(PyObject* obj) {
   // Some core types can be very quickly type-checked and do not allow NaN values
-#if PYARROW_IS_PY2
-  const int64_t non_nan_tpflags = Py_TPFLAGS_INT_SUBCLASS | Py_TPFLAGS_LONG_SUBCLASS |
-                                  Py_TPFLAGS_LIST_SUBCLASS | Py_TPFLAGS_TUPLE_SUBCLASS |
-                                  Py_TPFLAGS_STRING_SUBCLASS |
-                                  Py_TPFLAGS_UNICODE_SUBCLASS | Py_TPFLAGS_DICT_SUBCLASS |
-                                  Py_TPFLAGS_BASE_EXC_SUBCLASS | Py_TPFLAGS_TYPE_SUBCLASS;
-#else
   const int64_t non_nan_tpflags = Py_TPFLAGS_LONG_SUBCLASS | Py_TPFLAGS_LIST_SUBCLASS |
                                   Py_TPFLAGS_TUPLE_SUBCLASS | Py_TPFLAGS_BYTES_SUBCLASS |
                                   Py_TPFLAGS_UNICODE_SUBCLASS | Py_TPFLAGS_DICT_SUBCLASS |
                                   Py_TPFLAGS_BASE_EXC_SUBCLASS | Py_TPFLAGS_TYPE_SUBCLASS;
-#endif
   return !PyType_HasFeature(Py_TYPE(obj), non_nan_tpflags);
 }
 
@@ -313,10 +289,6 @@ Status UnboxIntegerAsInt64(PyObject* obj, int64_t* out) {
     if (overflow) {
       return Status::Invalid("PyLong is too large to fit int64");
     }
-#if PY_MAJOR_VERSION < 3
-  } else if (PyInt_Check(obj)) {
-    *out = static_cast<int64_t>(PyInt_AS_LONG(obj));
-#endif
   } else if (PyArray_IsScalar(obj, UByte)) {
     *out = reinterpret_cast<PyUByteScalarObject*>(obj)->obval;
   } else if (PyArray_IsScalar(obj, Short)) {

cpp/src/arrow/python/helpers.h

@@ -65,13 +65,7 @@ ARROW_PYTHON_EXPORT
 Status ImportFromModule(PyObject* module, const std::string& name, OwnedRef* ref);
 
 // \brief Check whether obj is an integer, independent of Python versions.
-inline bool IsPyInteger(PyObject* obj) {
-#if PYARROW_IS_PY2
-  return PyLong_Check(obj) || PyInt_Check(obj);
-#else
-  return PyLong_Check(obj);
-#endif
-}
+inline bool IsPyInteger(PyObject* obj) { return PyLong_Check(obj); }
 
 // \brief Use pandas missing value semantics to check if a value is null
 ARROW_PYTHON_EXPORT

cpp/src/arrow/python/io.cc

@@ -123,14 +123,6 @@ class PythonFile {
   Status Write(const std::shared_ptr<Buffer>& buffer) {
     RETURN_NOT_OK(CheckClosed());
 
-#if PY_MAJOR_VERSION < 3
-    // On Python 2, a write() method can typically call str() on its argument
-    // to get its bytes payload (this is the case with socket.makefile()).
-    // Unfortunately, on non-bytes buffer-like objects this will give out
-    // the repr() of the object rather than its data.  So fall back on
-    // copying the data to a bytes object.
-    return Write(buffer->data(), buffer->size());
-#else
     PyObject* py_data = wrap_buffer(buffer);
     PY_RETURN_IF_ERROR(StatusCode::IOError);
 
@@ -139,7 +131,6 @@ class PythonFile {
     Py_XDECREF(result);
     PY_RETURN_IF_ERROR(StatusCode::IOError);
     return Status::OK();
-#endif
   }
 
   Result<int64_t> Tell() {

cpp/src/arrow/python/numpy_internal.h

@@ -161,11 +161,6 @@ inline bool PyFloatScalar_Check(PyObject* obj) {
 }
 
 inline bool PyIntScalar_Check(PyObject* obj) {
-#if PY_MAJOR_VERSION < 3
-  if (PyInt_Check(obj)) {
-    return true;
-  }
-#endif
   return PyLong_Check(obj) || PyArray_IsScalar(obj, Integer);
 }
 

cpp/src/arrow/python/python_test.cc

@@ -85,9 +85,6 @@ TEST(OwnedRefNoGIL, TestMoves) {
 std::string FormatPythonException(const std::string& exc_class_name) {
   std::stringstream ss;
   ss << "Python exception: ";
-#if PY_MAJOR_VERSION < 3
-  ss << "exceptions.";
-#endif
   ss << exc_class_name;
   return ss.str();
 }

cpp/src/arrow/python/serialize.cc

@@ -103,11 +103,6 @@ class SequenceBuilder {
     return AppendPrimitive(&bools_, data, PythonType::BOOL);
   }
 
-  // Appending a python 2 int64_t to the sequence
-  Status AppendPy2Int64(const int64_t data) {
-    return AppendPrimitive(&py2_ints_, data, PythonType::PY2INT);
-  }
-
   // Appending an int64_t to the sequence
   Status AppendInt64(const int64_t data) {
     return AppendPrimitive(&ints_, data, PythonType::INT);
@@ -252,7 +247,6 @@ class SequenceBuilder {
 
   std::shared_ptr<BooleanBuilder> bools_;
   std::shared_ptr<Int64Builder> ints_;
-  std::shared_ptr<Int64Builder> py2_ints_;
   std::shared_ptr<BinaryBuilder> bytes_;
   std::shared_ptr<StringBuilder> strings_;
   std::shared_ptr<HalfFloatBuilder> half_floats_;
@@ -458,10 +452,6 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
       RETURN_NOT_OK(
           builder->AppendDict(context, serialized_object, recursion_depth, blobs_out));
     }
-#if PY_MAJOR_VERSION < 3
-  } else if (PyInt_Check(elem)) {
-    RETURN_NOT_OK(builder->AppendPy2Int64(static_cast<int64_t>(PyInt_AS_LONG(elem))));
-#endif
   } else if (PyBytes_Check(elem)) {
     auto data = reinterpret_cast<uint8_t*>(PyBytes_AS_STRING(elem));
     int32_t size = -1;

cpp/src/arrow/python/serialize.h

@@ -118,7 +118,7 @@ struct PythonType {
   enum type {
     BOOL,
     INT,
-    PY2INT,
+    PY2INT,  // Kept for compatibility
     BYTES,
     STRING,
     HALF_FLOAT,

dev/tasks/conda-recipes/.ci_support/linux_python2.7.yaml

@@ -1,52 +0,0 @@
-boost_cpp:
-- 1.70.0
-bzip2:
-- '1'
-c_compiler:
-- gcc
-c_compiler_version:
-- '7'
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '7'
-docker_image:
-- condaforge/linux-anvil-comp7
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.25'
-libprotobuf:
-- '3.11'
-lz4_c:
-- 1.8.3
-pin_run_as_build:
-  boost-cpp:
-    max_pin: x.x.x
-  bzip2:
-    max_pin: x
-  grpc-cpp:
-    max_pin: x.x
-  lz4-c:
-    max_pin: x.x.x
-  python:
-    min_pin: x.x
-    max_pin: x.x
-  snappy:
-    max_pin: x.x.x
-  zlib:
-    max_pin: x.x
-  zstd:
-    max_pin: x.x.x
-python:
-- '2.7'
-snappy:
-- 1.1.7
-zlib:
-- '1.2'
-zstd:
-- 1.4.4

dev/tasks/conda-recipes/.ci_support/osx_python2.7.yaml

@@ -1,56 +0,0 @@
-MACOSX_DEPLOYMENT_TARGET:
-- '10.9'
-boost_cpp:
-- 1.70.0
-bzip2:
-- '1'
-c_compiler:
-- clang
-c_compiler_version:
-- '9'
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cxx_compiler:
-- clangxx
-cxx_compiler_version:
-- '9'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.25'
-libprotobuf:
-- '3.11'
-lz4_c:
-- 1.8.3
-macos_machine:
-- x86_64-apple-darwin13.4.0
-macos_min_version:
-- '10.9'
-pin_run_as_build:
-  boost-cpp:
-    max_pin: x.x.x
-  bzip2:
-    max_pin: x
-  grpc-cpp:
-    max_pin: x.x
-  lz4-c:
-    max_pin: x.x.x
-  python:
-    min_pin: x.x
-    max_pin: x.x
-  snappy:
-    max_pin: x.x.x
-  zlib:
-    max_pin: x.x
-  zstd:
-    max_pin: x.x.x
-python:
-- '2.7'
-snappy:
-- 1.1.7
-zlib:
-- '1.2'
-zstd:
-- 1.4.4

dev/tasks/tasks.yml

@@ -438,35 +438,6 @@ tasks:
 
   ############################## Wheel Linux ##################################
 
-  wheel-manylinux1-cp27m:
-    ci: azure
-    platform: linux
-    template: python-wheels/azure.linux.yml
-    params:
-      python_version: 2.7
-      unicode_width: 16
-      wheel_tag: manylinux1
-      wheel_dir: manylinux1
-      test_docker_images: []
-      test_remove_system_libs: true
-    artifacts:
-      - pyarrow-{no_rc_version}-cp27-cp27m-manylinux1_x86_64.whl
-
-  wheel-manylinux1-cp27mu:
-    ci: azure
-    platform: linux
-    template: python-wheels/azure.linux.yml
-    params:
-      python_version: 2.7
-      unicode_width: 32
-      wheel_tag: manylinux1
-      wheel_dir: manylinux1
-      test_docker_images:
-        - python:2.7  # debian ucs4
-      test_remove_system_libs: false
-    artifacts:
-      - pyarrow-{no_rc_version}-cp27-cp27mu-manylinux1_x86_64.whl
-
   wheel-manylinux1-cp35m:
     ci: azure
     platform: linux
@@ -527,35 +498,6 @@ tasks:
     artifacts:
       - pyarrow-{no_rc_version}-cp38-cp38-manylinux1_x86_64.whl
 
-  wheel-manylinux2010-cp27m:
-    ci: azure
-    platform: linux
-    template: python-wheels/azure.linux.yml
-    params:
-      python_version: 2.7
-      unicode_width: 16
-      wheel_tag: manylinux2010
-      wheel_dir: manylinux201x
-      test_docker_images: []
-      test_remove_system_libs: true
-    artifacts:
-      - pyarrow-{no_rc_version}-cp27-cp27m-manylinux2010_x86_64.whl
-
-  wheel-manylinux2010-cp27mu:
-    ci: azure
-    platform: linux
-    template: python-wheels/azure.linux.yml
-    params:
-      python_version: 2.7
-      unicode_width: 32
-      wheel_tag: manylinux2010
-      wheel_dir: manylinux201x
-      test_docker_images:
-        - python:2.7  # debian ucs4
-      test_remove_system_libs: false
-    artifacts:
-      - pyarrow-{no_rc_version}-cp27-cp27mu-manylinux2010_x86_64.whl
-
   wheel-manylinux2010-cp35m:
     ci: azure
     platform: linux
@@ -678,15 +620,6 @@ tasks:
 
   ############################## Wheel OSX ####################################
 
-  wheel-osx-cp27m:
-    ci: travis
-    platform: osx
-    template: python-wheels/travis.osx.yml
-    params:
-      python_version: 2.7
-    artifacts:
-      - pyarrow-{no_rc_version}-cp27-cp27m-macosx_10_9_intel.whl
-
   wheel-osx-cp35m:
     ci: travis
     platform: osx

docs/source/python/install.rst

@@ -28,8 +28,7 @@ using a 64-bit system.
 Python Compatibility
 --------------------
 
-PyArrow pledges to maintain compatibility with Python 2.7 until the end of 2019.
-Other than that, PyArrow is currently compatible with Python 3.5, 3.6 and 3.7.
+PyArrow is currently compatible with Python 3.5, 3.6, 3.7 and 3.8.
 
 Using Conda
 -----------
@@ -55,11 +54,6 @@ If you encounter any importing issues of the pip wheels on Windows, you may
 need to install the `Visual C++ Redistributable for Visual Studio 2015
 <https://www.microsoft.com/en-us/download/details.aspx?id=48145>`_.
 
-.. note::
-
-   Windows packages are only available for Python 3.5 and higher (this is also
-   true for TensorFlow and any package that is implemented with modern C++).
-
 Installing from source
 ----------------------
 

python/manylinux1/README.md

@@ -31,14 +31,12 @@ for all supported Python versions and place them in the `dist` folder.
 
 ### Build instructions
 
-You can build the wheels with the following
-command (this is for Python 2.7 with unicode width 16, similarly you can pass
-in `PYTHON_VERSION="3.5"`, `PYTHON_VERSION="3.6"` or `PYTHON_VERSION="3.7"` or
-use `PYTHON_VERSION="2.7"` with `UNICODE_WIDTH=32`):
+You can build the wheels with the following command (this is for Python 3.7,
+similarly you can pass another value for `PYTHON_VERSION`):
 
 ```bash
 # Build the python packages
-docker-compose run -e PYTHON_VERSION="2.7" -e UNICODE_WIDTH=16 centos-python-manylinux1
+docker-compose run -e PYTHON_VERSION="3.7" centos-python-manylinux1
 # Now the new packages are located in the dist/ folder
 ls -l dist/
 ```
@@ -119,13 +117,11 @@ docker build -t arrow_linux_x86_64_base -f Dockerfile-x86_64_ubuntu .
 ```
 
 Once the image has been built, you can then build the wheels with the following
-command (this is for Python 2.7 with unicode width 16, similarly you can pass
-in `PYTHON_VERSION="3.5"`, `PYTHON_VERSION="3.6"` or `PYTHON_VERSION="3.7"` or
-use `PYTHON_VERSION="2.7"` with `UNICODE_WIDTH=32`)
+command:
 
 ```bash
 # Build the python packages
-sudo docker run --env UBUNTU_WHEELS=1 --env PYTHON_VERSION="2.7" --env UNICODE_WIDTH=16 --rm -t -i -v $PWD:/io -v $PWD/../../:/arrow arrow_linux_x86_64_base:latest /io/build_arrow.sh
+sudo docker run --env UBUNTU_WHEELS=1 --env PYTHON_VERSION="3.7" --rm -t -i -v $PWD:/io -v $PWD/../../:/arrow arrow_linux_x86_64_base:latest /io/build_arrow.sh
 # Now the new packages are located in the dist/ folder
 ls -l dist/
 echo "Please note that these wheels are not manylinux1 compliant"

python/manylinux1/build_arrow.sh

@@ -57,9 +57,10 @@ export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr -DBoost_NAMESPACE=arrow_boost -
 # Ensure the target directory exists
 mkdir -p /io/dist
 
-# Must pass PYTHON_VERSION and UNICODE_WIDTH env variables
-# possible values are: 2.7,16 2.7,32 3.5,16 3.6,16 3.7,16 3.8,16
+# Must pass PYTHON_VERSION env variable
+# possible values are: 3.5 3.6 3.7 3.8
 
+UNICODE_WIDTH=32  # Dummy value, irrelevant for Python 3
 CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION} ${UNICODE_WIDTH})"
 PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python"
 PIP="${CPYTHON_PATH}/bin/pip"
@@ -68,33 +69,19 @@ PATH="${CPYTHON_PATH}/bin:${PATH}"
 
 echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ==="
 $PIP install -r requirements-wheel.txt
-if [ "${PYTHON_VERSION}" = "2.7" -a "${UNICODE_WIDTH}" = "32" ]; then
-  # Can't use UNICODE_WIDTH in requirements.txt
-  $PIP install tensorflow
-fi
 
-if [ "${PYTHON_VERSION}" != "2.7" ]; then
-  export PYARROW_WITH_DATASET=1
-  export PYARROW_WITH_FLIGHT=1
-  export PYARROW_WITH_GANDIVA=1
-  export BUILD_ARROW_DATASET=ON
-  export BUILD_ARROW_FLIGHT=ON
-  export BUILD_ARROW_GANDIVA=ON
-else
-  # Flight and Gandiva are not supported on Python 2.7
-  export PYARROW_WITH_DATASET=0
-  export PYARROW_WITH_FLIGHT=0
-  export PYARROW_WITH_GANDIVA=0
-  export BUILD_ARROW_DATASET=OFF
-  export BUILD_ARROW_FLIGHT=OFF
-  export BUILD_ARROW_GANDIVA=OFF
-fi
+export PYARROW_WITH_DATASET=1
+export PYARROW_WITH_FLIGHT=1
+export PYARROW_WITH_GANDIVA=1
+export BUILD_ARROW_DATASET=ON
+export BUILD_ARROW_FLIGHT=ON
+export BUILD_ARROW_GANDIVA=ON
 
 # ARROW-3052(wesm): ORC is being bundled until it can be added to the
 # manylinux1 image
 
 echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
-ARROW_BUILD_DIR=/tmp/build-PY${PYTHON_VERSION}-${UNICODE_WIDTH}
+ARROW_BUILD_DIR=/tmp/build-PY${PYTHON_VERSION}
 mkdir -p "${ARROW_BUILD_DIR}"
 pushd "${ARROW_BUILD_DIR}"
 cmake \
@@ -164,15 +151,13 @@ else
   $PYTHON_INTERPRETER -c "
 import sys
 import pyarrow
-import pyarrow.parquet
-import pyarrow.plasma
+import pyarrow.dataset
+import pyarrow.flight
+import pyarrow.gandiva
 import pyarrow.fs
 import pyarrow._hdfs
-
-if sys.version_info.major > 2:
-    import pyarrow.dataset
-    import pyarrow.flight
-    import pyarrow.gandiva
+import pyarrow.parquet
+import pyarrow.plasma
   "
 
   # More thorough testing happens outside of the build to prevent

python/manylinux201x/README.md

@@ -36,19 +36,17 @@ The process is split up in two parts:
 ### Building PyArrow
 
 You can build the PyArrow wheels by running the following command in this
-directory (this is for Python 2.7 with unicode width 16, similarly you can pass
-in `PYTHON_VERSION="3.5"`, `PYTHON_VERSION="3.6"` or `PYTHON_VERSION="3.7"` or
-use `PYTHON_VERSION="2.7"` with `UNICODE_WIDTH=32`):
+directory (this is for Python 3.7, similarly you can pass another value
+for `PYTHON_VERSION`):
 
 ```bash
 # Build the python packages
-docker-compose run -e PYTHON_VERSION="2.7" -e UNICODE_WIDTH=16 centos-python-manylinux2010
+docker-compose run -e PYTHON_VERSION="3.7" centos-python-manylinux2010
 # Now the new packages are located in the dist/ folder
 ls -l dist/
 ```
 
 You can do the same for `manylinux2014` by substituting `centos-python-manylinux2014`.
-Note that `manylinux2014` does not support Python 2.7.
 
 ### Re-building the build image
 

python/manylinux201x/build_arrow.sh

@@ -58,10 +58,10 @@ export PYARROW_CMAKE_OPTIONS='-DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_
 # Ensure the target directory exists
 mkdir -p /io/dist
 
-# Must pass PYTHON_VERSION and UNICODE_WIDTH env variables
-# possible values are: 2.7,16 2.7,32 3.5,16 3.6,16 3.7,16 3.8,16
-# Note that manylinux2014 does not support Python 2.7
+# Must pass PYTHON_VERSION env variable
+# possible values are: 3.5 3.6 3.7 3.8
 
+UNICODE_WIDTH=32  # Dummy value, irrelevant for Python 3
 CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION} ${UNICODE_WIDTH})"
 PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python"
 PIP="${CPYTHON_PATH}/bin/pip"
@@ -69,33 +69,19 @@ PATH="${PATH}:${CPYTHON_PATH}"
 
 echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ==="
 $PIP install -r requirements-wheel.txt
-if [ "${PYTHON_VERSION}" = "2.7" -a "${UNICODE_WIDTH}" = "32" ]; then
-  # Can't use UNICODE_WIDTH in requirements.txt
-  $PIP install tensorflow
-fi
 
-if [ "${PYTHON_VERSION}" != "2.7" ]; then
-  export PYARROW_WITH_DATASET=1
-  export PYARROW_WITH_FLIGHT=1
-  export PYARROW_WITH_GANDIVA=1
-  export BUILD_ARROW_DATASET=ON
-  export BUILD_ARROW_FLIGHT=ON
-  export BUILD_ARROW_GANDIVA=ON
-else
-  # Flight and Gandiva are not supported on Python 2.7
-  export PYARROW_WITH_DATASET=0
-  export PYARROW_WITH_FLIGHT=0
-  export PYARROW_WITH_GANDIVA=0
-  export BUILD_ARROW_DATASET=OFF
-  export BUILD_ARROW_FLIGHT=OFF
-  export BUILD_ARROW_GANDIVA=OFF
-fi
+export PYARROW_WITH_DATASET=1
+export PYARROW_WITH_FLIGHT=1
+export PYARROW_WITH_GANDIVA=1
+export BUILD_ARROW_DATASET=ON
+export BUILD_ARROW_FLIGHT=ON
+export BUILD_ARROW_GANDIVA=ON
 
 # ARROW-3052(wesm): ORC is being bundled until it can be added to the
 # manylinux1 image
 
 echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
-ARROW_BUILD_DIR=/tmp/build-PY${PYTHON_VERSION}-${UNICODE_WIDTH}
+ARROW_BUILD_DIR=/tmp/build-PY${PYTHON_VERSION}
 mkdir -p "${ARROW_BUILD_DIR}"
 pushd "${ARROW_BUILD_DIR}"
 PATH="${CPYTHON_PATH}/bin:${PATH}" cmake \
@@ -160,17 +146,14 @@ $PIP install repaired_wheels/*.whl
 
 # Test that the modules are importable
 $PYTHON_INTERPRETER -c "
-import sys
 import pyarrow
-import pyarrow.parquet
-import pyarrow.plasma
+import pyarrow.dataset
+import pyarrow.flight
+import pyarrow.gandiva
 import pyarrow.fs
 import pyarrow._hdfs
-
-if sys.version_info.major > 2:
-    import pyarrow.dataset
-    import pyarrow.flight
-    import pyarrow.gandiva
+import pyarrow.parquet
+import pyarrow.plasma
 "
 
 # More thorough testing happens outside of the build to prevent

python/pyarrow/__init__.pxd

@@ -15,8 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
-
 from libcpp.memory cimport shared_ptr
 from pyarrow.includes.libarrow cimport (CArray, CBuffer, CDataType,
                                         CField, CRecordBatch, CSchema,

python/pyarrow/__init__.py

@@ -17,7 +17,6 @@
 
 # flake8: noqa
 
-from __future__ import absolute_import
 
 import os as _os
 import sys as _sys
@@ -261,7 +260,7 @@ def get_library_dirs():
                 if not library_dir.startswith("-L"):
                     raise ValueError(
                         "pkg-config --libs-only-L returned unexpected "
-                        "value {0!r}".format(library_dir))
+                        "value {!r}".format(library_dir))
                 append_library_dir(library_dir[2:])
 
     if _sys.platform == 'win32':

python/pyarrow/_csv.pyx

@@ -20,7 +20,6 @@
 # cython: embedsignature = True
 # cython: language_level = 3
 
-from __future__ import absolute_import
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *

python/pyarrow/_cuda.pxd

@@ -17,8 +17,6 @@
 
 # cython: language_level = 3
 
-from __future__ import absolute_import
-
 from pyarrow.lib cimport *
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *

python/pyarrow/_cuda.pyx

@@ -16,8 +16,6 @@
 # under the License.
 
 
-from __future__ import absolute_import
-
 from pyarrow.compat import tobytes
 from pyarrow.lib cimport *
 from pyarrow.includes.libarrow_cuda cimport *
@@ -687,14 +685,6 @@ cdef class CudaBuffer(Buffer):
         # cannot support buffer protocol PEP-3118 for CudaBuffer.
         raise BufferError('buffer protocol for device buffer not supported')
 
-    def __getreadbuffer__(self, Py_ssize_t idx, void** p):
-        # Python 2.x specific method
-        raise NotImplementedError('CudaBuffer.__getreadbuffer__')
-
-    def __getwritebuffer__(self, Py_ssize_t idx, void** p):
-        # Python 2.x specific method
-        raise NotImplementedError('CudaBuffer.__getwritebuffer__')
-
 
 cdef class HostBuffer(Buffer):
     """Device-accessible CPU memory created using cudaHostAlloc.

python/pyarrow/_dataset.pyx

@@ -19,9 +19,6 @@
 
 """Dataset is currently unstable. APIs subject to change without notice."""
 
-from __future__ import absolute_import
-
-import six
 from cpython.object cimport Py_LT, Py_EQ, Py_GT, Py_LE, Py_NE, Py_GE
 from cython.operator cimport dereference as deref
 
@@ -1274,7 +1271,7 @@ cdef class ScalarExpression(Expression):
             scalar = MakeScalar(<double>value)
         elif isinstance(value, int):
             scalar = MakeScalar(<int64_t>value)
-        elif isinstance(value, six.string_types):
+        elif isinstance(value, (bytes, str)):
             scalar = MakeStringScalar(tobytes(value))
         else:
             raise TypeError('Not yet supported scalar value: {}'.format(value))

python/pyarrow/_flight.pyx

@@ -18,8 +18,6 @@
 # cython: language_level = 3
 # cython: embedsignature = True
 
-from __future__ import absolute_import
-
 import collections
 import contextlib
 import enum
@@ -29,8 +27,6 @@ import time
 import threading
 import warnings
 
-import six
-
 from cython.operator cimport dereference as deref
 from cython.operator cimport postincrement
 
@@ -551,7 +547,7 @@ cdef class Location:
     @staticmethod
     cdef CLocation unwrap(object location) except *:
         cdef CLocation c_location
-        if isinstance(location, six.text_type):
+        if isinstance(location, str):
             check_flight_status(
                 CLocation.Parse(tobytes(location), &c_location))
             return c_location
@@ -923,7 +919,7 @@ cdef class FlightClient:
 
     def __init__(self, location, tls_root_certs=None, override_hostname=None,
                  middleware=None):
-        if isinstance(location, six.string_types):
+        if isinstance(location, (bytes, str)):
             location = Location(location)
         elif isinstance(location, tuple):
             host, port = location
@@ -2100,7 +2096,7 @@ cdef class _ServerMiddlewareWrapper(ServerMiddleware):
             # Manually merge with existing headers (since headers are
             # multi-valued)
             for key, values in more_headers.items():
-                if isinstance(values, (six.text_type, six.binary_type)):
+                if isinstance(values, (bytes, str)):
                     values = (values,)
                 headers[key].extend(values)
         return headers
@@ -2138,7 +2134,7 @@ cdef class FlightServerBase:
 
     def __init__(self, location=None, auth_handler=None,
                  tls_certificates=None, middleware=None):
-        if isinstance(location, six.string_types):
+        if isinstance(location, (bytes, str)):
             location = Location(location)
         elif isinstance(location, (tuple, type(None))):
             if location is None:

python/pyarrow/_fs.pxd

@@ -17,8 +17,6 @@
 
 # cython: language_level = 3
 
-import six
-
 from pyarrow.compat import frombytes, tobytes
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport PyDateTime_from_TimePoint

python/pyarrow/_fs.pyx

@@ -17,7 +17,6 @@
 
 # cython: language_level = 3
 
-import six
 from cpython.datetime cimport datetime, PyDateTime_DateTime
 
 from pyarrow.compat import frombytes, tobytes
@@ -32,7 +31,7 @@ from pyarrow.lib cimport *
 cdef inline c_string _path_as_bytes(path) except *:
     # handle only abstract paths, not bound to any filesystem like pathlib is,
     # so we only accept plain strings
-    if not isinstance(path, six.string_types):
+    if not isinstance(path, (bytes, str)):
         raise TypeError('Path must be a string')
     # tobytes always uses utf-8, which is more or less ok, at least on Windows
     # since the C++ side then decodes from utf-8. On Unix, os.fsencode may be

python/pyarrow/_hdfs.pyx

@@ -17,8 +17,6 @@
 
 # cython: language_level = 3
 
-import six
-
 from pyarrow.lib cimport check_status
 from pyarrow.compat import frombytes, tobytes
 from pyarrow.includes.common cimport *
@@ -167,7 +165,7 @@ cdef class HadoopFileSystem(FileSystem):
             HdfsOptions options
             shared_ptr[CHadoopFileSystem] wrapped
 
-        if isinstance(options_or_uri, six.string_types):
+        if isinstance(options_or_uri, str):
             options = HdfsOptions.from_uri(options_or_uri)
         elif isinstance(options_or_uri, HdfsOptions):
             options = options_or_uri

python/pyarrow/_json.pyx

@@ -20,8 +20,6 @@
 # cython: embedsignature = True
 # cython: language_level = 3
 
-from __future__ import absolute_import
-
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
 from pyarrow.lib cimport (check_status, Field, MemoryPool, ensure_type,

python/pyarrow/_orc.pxd

@@ -18,8 +18,6 @@
 # distutils: language = c++
 # cython: language_level = 3
 
-from __future__ import absolute_import
-
 from libc.string cimport const_char
 from libcpp.vector cimport vector as std_vector
 from pyarrow.includes.common cimport *

python/pyarrow/_orc.pyx

@@ -19,8 +19,6 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-from __future__ import absolute_import
-
 from cython.operator cimport dereference as deref
 from libcpp.vector cimport vector as std_vector
 from pyarrow.includes.common cimport *
@@ -32,7 +30,6 @@ from pyarrow.lib cimport (check_status,
                           RecordBatch,
                           pyarrow_wrap_table,
                           get_reader)
-import six
 
 
 cdef class ORCReader:

python/pyarrow/_parquet.pxd

@@ -18,8 +18,6 @@
 # distutils: language = c++
 # cython: language_level = 3
 
-from __future__ import absolute_import
-
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport (CChunkedArray, CSchema, CStatus,
                                         CTable, CMemoryPool, CBuffer,

python/pyarrow/_parquet.pyx

@@ -19,10 +19,8 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-from __future__ import absolute_import
-
 import io
-import six
+from textwrap import indent
 import warnings
 
 import numpy as np
@@ -46,7 +44,6 @@ from pyarrow.lib import (ArrowException, NativeFile, _stringify_path,
                          _datetime_conversion_functions,
                          _box_time_milli,
                          _box_time_micro)
-from pyarrow.util import indent
 
 cimport cpython as cp
 

python/pyarrow/_plasma.pyx

@@ -20,8 +20,6 @@
 # cython: embedsignature = True
 # cython: language_level = 3
 
-from __future__ import absolute_import
-
 from libcpp cimport bool as c_bool, nullptr
 from libcpp.memory cimport shared_ptr, unique_ptr, make_shared
 from libcpp.string cimport string as c_string

python/pyarrow/_s3fs.pyx

@@ -17,8 +17,6 @@
 
 # cython: language_level = 3
 
-import six
-
 from pyarrow.lib cimport check_status
 from pyarrow.compat import frombytes, tobytes
 from pyarrow.includes.common cimport *

python/pyarrow/array.pxi

@@ -1848,11 +1848,11 @@ cdef class StructArray(Array):
             CStructArray* arr = <CStructArray*> self.ap
             shared_ptr[CArray] child
 
-        if isinstance(index, six.string_types):
+        if isinstance(index, (bytes, str)):
             child = arr.GetFieldByName(tobytes(index))
             if child == nullptr:
                 raise KeyError(index)
-        elif isinstance(index, six.integer_types):
+        elif isinstance(index, int):
             child = arr.field(
                 <int>_normalize_index(index, self.ap.num_fields()))
         else:

python/pyarrow/benchmark.py

@@ -17,6 +17,5 @@
 
 # flake8: noqa
 
-from __future__ import absolute_import
 
 from pyarrow.lib import benchmark_PandasObjectIsNull

python/pyarrow/builder.pxi

@@ -15,8 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import six
-
 from pyarrow.compat import tobytes
 
 
@@ -45,7 +43,7 @@ cdef class StringBuilder:
         """
         if value is None or value is np.nan:
             self.builder.get().AppendNull()
-        elif isinstance(value, (six.string_types, six.binary_type)):
+        elif isinstance(value, (bytes, str)):
             self.builder.get().Append(tobytes(value))
         else:
             raise TypeError('StringBuilder only accepts string objects')

python/pyarrow/compat.py

@@ -17,114 +17,31 @@
 
 # flake8: noqa
 
-from __future__ import absolute_import
-
-import itertools
+import sys
+import socket
+import unittest
 
 import numpy as np
 
-import sys
-import six
-from six import BytesIO, StringIO, string_types as py_string
-import socket
+try:
+    import pickle5 as builtin_pickle
+except ImportError:
+    import pickle as builtin_pickle
 
+from collections.abc import Iterable, Mapping, Sequence
 
-PY26 = sys.version_info[:2] == (2, 6)
-PY2 = sys.version_info[0] == 2
+def guid():
+    from uuid import uuid4
+    return uuid4().hex
 
-
-if PY26:
-    import unittest2 as unittest
-else:
-    import unittest
-
-
-if PY2:
-    import cPickle as builtin_pickle
-
-    try:
-        from cdecimal import Decimal
-    except ImportError:
-        from decimal import Decimal
-
-    from collections import Iterable, Mapping, Sequence
-
-    unicode_type = unicode
-    file_type = file
-    lzip = zip
-    zip = itertools.izip
-    zip_longest = itertools.izip_longest
-
-    def dict_values(x):
-        return x.values()
-
-    range = xrange
-    long = long
-
-    def guid():
-        from uuid import uuid4
-        return uuid4().get_hex()
-
-    def u(s):
-        return unicode(s, "unicode_escape")
-
-    def tobytes(o):
-        if isinstance(o, unicode):
-            return o.encode('utf8')
-        else:
-            return o
-
-    def u_utf8(s):
-        return s.decode('utf-8')
-
-    def frombytes(o):
+def tobytes(o):
+    if isinstance(o, str):
+        return o.encode('utf8')
+    else:
         return o
 
-    def unichar(s):
-        return unichr(s)
-else:
-    try:
-        import pickle5 as builtin_pickle
-    except ImportError:
-        import pickle as builtin_pickle
-
-    from collections.abc import Iterable, Mapping, Sequence
-
-    unicode_type = str
-    file_type = None
-    def lzip(*x):
-        return list(zip(*x))
-    long = int
-    zip = zip
-    zip_longest = itertools.zip_longest
-    def dict_values(x):
-        return list(x.values())
-    from decimal import Decimal
-    range = range
-
-    def guid():
-        from uuid import uuid4
-        return uuid4().hex
-
-    def u(s):
-        return s
-
-    def tobytes(o):
-        if isinstance(o, str):
-            return o.encode('utf8')
-        else:
-            return o
-
-    def u_utf8(s):
-        if isinstance(s, bytes):
-            return frombytes(s)
-        return s
-
-    def frombytes(o):
-        return o.decode('utf8')
-
-    def unichar(s):
-        return chr(s)
+def frombytes(o):
+    return o.decode('utf8')
 
 
 if sys.version_info >= (3, 7):
@@ -141,8 +58,7 @@ except ImportError:
     pickle = builtin_pickle
 
 def encode_file_path(path):
-    import os
-    if isinstance(path, unicode_type):
+    if isinstance(path, str):
         # POSIX systems can handle utf-8. UTF8 is converted to utf16-le in
         # libarrow
         encoded_path = path.encode('utf-8')
@@ -154,15 +70,11 @@ def encode_file_path(path):
     return encoded_path
 
 
-integer_types = six.integer_types + (np.integer,)
+integer_types = (int, np.integer,)
 
 
 def get_socket_from_fd(fileno, family, type):
-    if PY2:
-        socket_obj = socket.fromfd(fileno, family, type)
-        return socket.socket(family, type, _sock=socket_obj)
-    else:
-        return socket.socket(fileno=fileno, family=family, type=type)
+    return socket.socket(fileno=fileno, family=family, type=type)
 
 
 try:

python/pyarrow/compute.py

@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
 
 from pyarrow._compute import (  # noqa
     sum

python/pyarrow/csv.py

@@ -15,6 +15,5 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
 
 from pyarrow._csv import ReadOptions, ParseOptions, ConvertOptions, read_csv  # noqa

python/pyarrow/cuda.py

@@ -17,7 +17,6 @@
 
 # flake8: noqa
 
-from __future__ import absolute_import
 
 from pyarrow._cuda import (Context, IpcMemHandle, CudaBuffer,
                            HostBuffer, BufferReader, BufferWriter,

python/pyarrow/dataset.py

@@ -17,16 +17,9 @@
 
 """Dataset is currently unstable. APIs subject to change without notice."""
 
-from __future__ import absolute_import
-
-import sys
-
 import pyarrow as pa
 from pyarrow.util import _stringify_path, _is_path_like
 
-if sys.version_info < (3,):
-    raise ImportError("Python Dataset bindings require Python 3")
-
 from pyarrow._dataset import (  # noqa
     AndExpression,
     CastExpression,
@@ -134,7 +127,7 @@ def partitioning(schema=None, field_names=None, flavor=None):
                 return DirectoryPartitioning.discover(field_names)
             else:
                 raise ValueError(
-                    "Expected list of field names, got {0}".format(
+                    "Expected list of field names, got {}".format(
                         type(field_names)))
         else:
             raise ValueError(
@@ -148,7 +141,7 @@ def partitioning(schema=None, field_names=None, flavor=None):
                 return HivePartitioning(schema)
             else:
                 raise ValueError(
-                    "Expected Schema for 'schema', got {0}".format(
+                    "Expected Schema for 'schema', got {}".format(
                         type(schema)))
         else:
             return HivePartitioning.discover()
@@ -207,7 +200,7 @@ def _ensure_partitioning(scheme):
         pass
     else:
         ValueError(
-            "Expected Partitioning or PartitioningFactory, got {0}".format(
+            "Expected Partitioning or PartitioningFactory, got {}".format(
                 type(scheme)))
     return scheme
 
@@ -218,7 +211,7 @@ def _ensure_format(obj):
     elif obj == "parquet":
         return ParquetFileFormat()
     else:
-        raise ValueError("format '{0}' is not supported".format(obj))
+        raise ValueError("format '{}' is not supported".format(obj))
 
 
 def source(path_or_paths, filesystem=None, partitioning=None,
@@ -280,7 +273,7 @@ def _ensure_source(src, **kwargs):
         )
     else:
         raise TypeError(
-            "Expected a path-like or Source, got {0}".format(type(src))
+            "Expected a path-like or Source, got {}".format(type(src))
         )
 
 

python/pyarrow/feather.py

@@ -15,12 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
 
 import os
 
-import six
-
 from pyarrow.pandas_compat import _pandas_api  # noqa
 from pyarrow.lib import FeatherError  # noqa
 from pyarrow.lib import Table, concat_tables
@@ -63,17 +60,17 @@ def check_chunked_overflow(name, col):
         return
 
     if col.type in (ext.binary(), ext.string()):
-        raise ValueError("Column '{0}' exceeds 2GB maximum capacity of "
+        raise ValueError("Column '{}' exceeds 2GB maximum capacity of "
                          "a Feather binary column. This restriction may be "
                          "lifted in the future".format(name))
     else:
         # TODO(wesm): Not sure when else this might be reached
-        raise ValueError("Column '{0}' of type {1} was chunked on conversion "
+        raise ValueError("Column '{}' of type {} was chunked on conversion "
                          "to Arrow and cannot be currently written to "
                          "Feather format".format(name, str(col.type)))
 
 
-class FeatherWriter(object):
+class FeatherWriter:
 
     def __init__(self, dest):
         _check_pandas_version()
@@ -100,7 +97,7 @@ class FeatherWriter(object):
         self.writer.close()
 
 
-class FeatherDataset(object):
+class FeatherDataset:
     """
     Encapsulates details of reading a list of Feather files.
 
@@ -143,8 +140,8 @@ class FeatherDataset(object):
 
     def validate_schemas(self, piece, table):
         if not self.schema.equals(table.schema):
-            raise ValueError('Schema in {0!s} was different. \n'
-                             '{1!s}\n\nvs\n\n{2!s}'
+            raise ValueError('Schema in {!s} was different. \n'
+                             '{!s}\n\nvs\n\n{!s}'
                              .format(piece, self.schema,
                                      table.schema))
 
@@ -186,7 +183,7 @@ def write_feather(df, dest):
         import gc
         writer = None
         gc.collect()
-        if isinstance(dest, six.string_types):
+        if isinstance(dest, str):
             try:
                 os.remove(dest)
             except os.error:

python/pyarrow/filesystem.py

@@ -15,20 +15,19 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
 
 import os
 import inspect
 import posixpath
+import urllib.parse
 
 from os.path import join as pjoin
-from six.moves.urllib.parse import urlparse
 
 import pyarrow as pa
 from pyarrow.util import implements, _stringify_path, _is_path_like
 
 
-class FileSystem(object):
+class FileSystem:
     """
     Abstract filesystem interface
     """
@@ -369,8 +368,7 @@ class S3FSWrapper(DaskFileSystem):
         yield path, directories, files
 
         for directory in directories:
-            for tup in self.walk(directory, refresh=refresh):
-                yield tup
+            yield from self.walk(directory, refresh=refresh)
 
 
 def _sanitize_s3(path):
@@ -394,7 +392,7 @@ def _ensure_filesystem(fs):
             elif mro.__name__ == 'LocalFileSystem':
                 return LocalFileSystem.get_instance()
 
-        raise IOError('Unrecognized filesystem: {0}'.format(fs_type))
+        raise OSError('Unrecognized filesystem: {}'.format(fs_type))
     else:
         return fs
 
@@ -415,7 +413,7 @@ def resolve_filesystem_and_path(where, filesystem=None):
     if filesystem is not None:
         return _ensure_filesystem(filesystem), path
 
-    parsed_uri = urlparse(path)
+    parsed_uri = urllib.parse.urlparse(path)
     if parsed_uri.scheme == 'hdfs' or parsed_uri.scheme == 'viewfs':
         # Input is hdfs URI such as hdfs://host:port/myfile.parquet
         netloc_split = parsed_uri.netloc.split(':')

python/pyarrow/flight.py

@@ -15,13 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
-
-import sys
-
-if sys.version_info < (3,):
-    raise ImportError("Python Flight bindings require Python 3")
-
 from pyarrow._flight import (  # noqa
     connect,
     Action,

python/pyarrow/fs.py

@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
 
 from pyarrow._fs import (  # noqa
     FileSelector,

python/pyarrow/gandiva.pyx

@@ -20,8 +20,6 @@
 # cython: embedsignature = True
 # cython: language_level = 3
 
-from __future__ import absolute_import
-
 from libcpp cimport bool as c_bool, nullptr
 from libcpp.memory cimport shared_ptr, unique_ptr, make_shared
 from libcpp.string cimport string as c_string

python/pyarrow/hdfs.py

@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
 
 import os
 import posixpath
@@ -53,15 +52,15 @@ class HadoopFileSystem(lib.HadoopFileSystem, FileSystem):
 
     @implements(FileSystem.isdir)
     def isdir(self, path):
-        return super(HadoopFileSystem, self).isdir(path)
+        return super().isdir(path)
 
     @implements(FileSystem.isfile)
     def isfile(self, path):
-        return super(HadoopFileSystem, self).isfile(path)
+        return super().isfile(path)
 
     @implements(FileSystem.delete)
     def delete(self, path, recursive=False):
-        return super(HadoopFileSystem, self).delete(path, recursive)
+        return super().delete(path, recursive)
 
     def mkdir(self, path, **kwargs):
         """
@@ -76,15 +75,15 @@ class HadoopFileSystem(lib.HadoopFileSystem, FileSystem):
         -----
         libhdfs does not support create_parents=False, so we ignore this here
         """
-        return super(HadoopFileSystem, self).mkdir(path)
+        return super().mkdir(path)
 
     @implements(FileSystem.rename)
     def rename(self, path, new_path):
-        return super(HadoopFileSystem, self).rename(path, new_path)
+        return super().rename(path, new_path)
 
     @implements(FileSystem.exists)
     def exists(self, path):
-        return super(HadoopFileSystem, self).exists(path)
+        return super().exists(path)
 
     def ls(self, path, detail=False):
         """
@@ -100,7 +99,7 @@ class HadoopFileSystem(lib.HadoopFileSystem, FileSystem):
         -------
         result : list of dicts (detail=True) or strings (detail=False)
         """
-        return super(HadoopFileSystem, self).ls(path, detail)
+        return super().ls(path, detail)
 
     def walk(self, top_path):
         """
@@ -120,8 +119,7 @@ class HadoopFileSystem(lib.HadoopFileSystem, FileSystem):
         directories, files = _libhdfs_walk_files_dirs(top_path, contents)
         yield top_path, directories, files
         for dirname in directories:
-            for tup in self.walk(self._path_join(top_path, dirname)):
-                yield tup
+            yield from self.walk(self._path_join(top_path, dirname))
 
 
 def _maybe_set_hadoop_classpath():
@@ -134,7 +132,7 @@ def _maybe_set_hadoop_classpath():
         if sys.platform != 'win32':
             classpath = _derive_hadoop_classpath()
         else:
-            hadoop_bin = '{0}/bin/hadoop'.format(os.environ['HADOOP_HOME'])
+            hadoop_bin = '{}/bin/hadoop'.format(os.environ['HADOOP_HOME'])
             classpath = _hadoop_classpath_glob(hadoop_bin)
     else:
         classpath = _hadoop_classpath_glob('hadoop')

python/pyarrow/io-hdfs.pxi

@@ -18,13 +18,10 @@
 # ----------------------------------------------------------------------
 # HDFS IO implementation
 
-_HDFS_PATH_RE = re.compile(r'hdfs://(.*):(\d+)(.*)')
+from queue import Queue, Empty as QueueEmpty, Full as QueueFull
 
-try:
-    # Python 3
-    from queue import Queue, Empty as QueueEmpty, Full as QueueFull
-except ImportError:
-    from Queue import Queue, Empty as QueueEmpty, Full as QueueFull
+
+_HDFS_PATH_RE = re.compile(r'hdfs://(.*):(\d+)(.*)')
 
 
 def have_libhdfs():

python/pyarrow/io.pxi

@@ -21,7 +21,6 @@
 from libc.stdlib cimport malloc, free
 
 import re
-import six
 import sys
 import threading
 import time
@@ -30,7 +29,7 @@ from io import BufferedIOBase, IOBase, TextIOBase, UnsupportedOperation
 
 from pyarrow.util import _stringify_path
 from pyarrow.compat import (
-    builtin_pickle, frombytes, tobytes, encode_file_path, file_type)
+    builtin_pickle, frombytes, tobytes, encode_file_path)
 
 
 # 64K
@@ -665,14 +664,6 @@ cdef class PythonFile(NativeFile):
                 else:
                     if not handle.writable():
                         raise TypeError("writable file expected")
-            elif file_type is not None and isinstance(handle, file_type):
-                # Python 2 file type
-                if kind == 'r':
-                    if 'r' not in handle.mode and '+' not in handle.mode:
-                        raise TypeError("readable file expected")
-                else:
-                    if 'w' not in handle.mode and '+' not in handle.mode:
-                        raise TypeError("writable file expected")
             # (other duck-typed file-like objects are possible)
 
         # If possible, check the file is a binary file
@@ -1482,7 +1473,7 @@ cdef CompressionType _get_compression_type(object name) except *:
 
 
 def _detect_compression(path):
-    if isinstance(path, six.string_types):
+    if isinstance(path, str):
         if path.endswith('.bz2'):
             return 'bz2'
         elif path.endswith('.gz'):

python/pyarrow/ipc.py

@@ -17,7 +17,6 @@
 
 # Arrow file and stream reader/writer classes, and other messaging tools
 
-from __future__ import absolute_import
 
 import pyarrow as pa
 
@@ -28,7 +27,7 @@ from pyarrow.lib import (Message, MessageReader,  # noqa
 import pyarrow.lib as lib
 
 
-class _ReadPandasOption(object):
+class _ReadPandasOption:
 
     def read_pandas(self, **options):
         """

python/pyarrow/json.py

@@ -15,6 +15,5 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
 
 from pyarrow._json import ReadOptions, ParseOptions, read_json  # noqa

python/pyarrow/jvm.py

@@ -25,9 +25,6 @@ through jpype. Modules that talk to a remote JVM like py4j will not work as the
 memory addresses reported by them are not reachable in the python process.
 """
 
-from __future__ import absolute_import
-
-
 import pyarrow as pa
 
 

python/pyarrow/lib.pxd

@@ -17,8 +17,6 @@
 
 # cython: language_level = 3
 
-from __future__ import absolute_import
-
 from cpython cimport PyObject
 from libcpp cimport nullptr
 from libcpp.cast cimport dynamic_cast

python/pyarrow/lib.pyx

@@ -19,14 +19,11 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-from __future__ import absolute_import
-
 import datetime
 import decimal as _pydecimal
 import json
 import numpy as np
 import os
-import six
 
 from pyarrow.compat import frombytes, tobytes, ordered_dict
 

python/pyarrow/orc.py

@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
 
 from itertools import count
 from numbers import Integral
@@ -41,14 +40,12 @@ def _traverse(typ, counter):
             for sub, c in _traverse(field.type, counter):
                 yield path + sub, c
     elif _is_map(typ):
-        for sub_c in _traverse(typ.value_type, counter):
-            yield sub_c
+        yield from _traverse(typ.value_type, counter)
     elif types.is_list(typ):
         # Skip one index for list type, since this can never be selected
         # directly
         next(counter)
-        for sub_c in _traverse(typ.value_type, counter):
-            yield sub_c
+        yield from _traverse(typ.value_type, counter)
     elif types.is_union(typ):
         # Union types not supported, just skip the indexes
         for dtype in typ:
@@ -61,7 +58,7 @@ def _schema_to_indices(schema):
     return {'.'.join(i): c for i, c in _traverse(schema, count(1))}
 
 
-class ORCFile(object):
+class ORCFile:
     """
     Reader interface for a single ORC file
 

python/pyarrow/pandas_compat.py

@@ -15,9 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
 
 import ast
+from itertools import zip_longest
 import json
 import operator
 import re
@@ -26,12 +26,10 @@ from copy import deepcopy
 
 import numpy as np
 
-import six
-
 import pyarrow as pa
 from pyarrow.lib import _pandas_api
 from pyarrow.compat import (builtin_pickle,  # noqa
-                            PY2, zip_longest, Sequence, u_utf8)
+                            frombytes, Sequence)
 
 
 _logical_type_map = {}
@@ -96,8 +94,8 @@ _numpy_logical_type_map = {
     np.float32: 'float32',
     np.float64: 'float64',
     'datetime64[D]': 'date',
-    np.unicode_: 'string' if not PY2 else 'unicode',
-    np.bytes_: 'bytes' if not PY2 else 'string',
+    np.unicode_: 'string',
+    np.bytes_: 'bytes',
 }
 
 
@@ -112,7 +110,7 @@ def get_logical_type_from_numpy(pandas_collection):
             return 'datetime64[ns]'
         result = _pandas_api.infer_dtype(pandas_collection)
         if result == 'string':
-            return 'bytes' if PY2 else 'unicode'
+            return 'unicode'
         return result
 
 
@@ -162,14 +160,14 @@ def get_column_metadata(column, name, arrow_type, field_name):
         }
         string_dtype = 'object'
 
-    if name is not None and not isinstance(name, six.string_types):
+    if name is not None and not isinstance(name, str):
         raise TypeError(
             'Column name must be a string. Got column {} of type {}'.format(
                 name, type(name).__name__
             )
         )
 
-    assert field_name is None or isinstance(field_name, six.string_types), \
+    assert field_name is None or isinstance(field_name, str), \
         str(type(field_name))
     return {
         'name': name,
@@ -295,9 +293,9 @@ def _column_name_to_strings(name):
     >>> _column_name_to_strings(name)
     ('1', '2017-02-01 00:00:00')
     """
-    if isinstance(name, six.string_types):
+    if isinstance(name, str):
         return name
-    elif isinstance(name, six.binary_type):
+    elif isinstance(name, bytes):
         # XXX: should we assume that bytes in Python 3 are UTF-8?
         return name.decode('utf8')
     elif isinstance(name, tuple):
@@ -561,7 +559,7 @@ def dataframe_to_arrays(df, schema, preserve_index, nthreads=1, columns=None,
         except (pa.ArrowInvalid,
                 pa.ArrowNotImplementedError,
                 pa.ArrowTypeError) as e:
-            e.args += ("Conversion failed for column {0!s} with type {1!s}"
+            e.args += ("Conversion failed for column {!s} with type {!s}"
                        .format(col.name, col.dtype),)
             raise e
         if not field_nullable and result.null_count > 0:
@@ -771,11 +769,11 @@ def table_to_blockmanager(options, table, categories=None,
 
 # Set of the string repr of all numpy dtypes that can be stored in a pandas
 # dataframe (complex not included since not supported by Arrow)
-_pandas_supported_numpy_types = set([
+_pandas_supported_numpy_types = {
     str(np.dtype(typ))
     for typ in (np.sctypes['int'] + np.sctypes['uint'] + np.sctypes['float']
                 + ['object', 'bool'])
-])
+}
 
 
 def _get_extension_dtypes(table, columns_metadata, types_mapper=None):
@@ -843,7 +841,8 @@ def _check_data_column_metadata_consistency(all_columns):
 
 
 def _deserialize_column_index(block_table, all_columns, column_indexes):
-    column_strings = [u_utf8(x) for x in block_table.column_names]
+    column_strings = [frombytes(x) if isinstance(x, bytes) else x
+                      for x in block_table.column_names]
     if all_columns:
         columns_name_dict = {
             c.get('field_name', _column_name_to_strings(c['name'])): c['name']
@@ -898,7 +897,7 @@ def _reconstruct_index(table, index_descriptors, all_columns):
     index_names = []
     result_table = table
     for descr in index_descriptors:
-        if isinstance(descr, six.string_types):
+        if isinstance(descr, str):
             result_table, index_level, index_name = _extract_index_level(
                 table, result_table, descr, field_name_to_metadata)
             if index_level is None:
@@ -914,7 +913,7 @@ def _reconstruct_index(table, index_descriptors, all_columns):
                 # Possibly the result of munged metadata
                 continue
         else:
-            raise ValueError("Unrecognized index kind: {0}"
+            raise ValueError("Unrecognized index kind: {}"
                              .format(descr['kind']))
         index_arrays.append(index_level)
         index_names.append(index_name)
@@ -1128,7 +1127,7 @@ def _add_any_metadata(table, pandas_metadata):
     index_columns = pandas_metadata['index_columns']
     # only take index columns into account if they are an actual table column
     index_columns = [idx_col for idx_col in index_columns
-                     if isinstance(idx_col, six.string_types)]
+                     if isinstance(idx_col, str)]
     n_index_levels = len(index_columns)
     n_columns = len(pandas_metadata['columns']) - n_index_levels
 

python/pyarrow/parquet.py

@@ -15,19 +15,17 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
 
 from collections import defaultdict
 from concurrent import futures
 from functools import partial
 
-from six.moves.urllib.parse import urlparse
 import json
 import numpy as np
 import os
 import re
-import six
 import operator
+import urllib.parse
 
 import pyarrow as pa
 import pyarrow.lib as lib
@@ -47,7 +45,7 @@ _URI_STRIP_SCHEMES = ('hdfs',)
 
 def _parse_uri(path):
     path = _stringify_path(path)
-    parsed_uri = urlparse(path)
+    parsed_uri = urllib.parse.urlparse(path)
     if parsed_uri.scheme in _URI_STRIP_SCHEMES:
         return parsed_uri.path
     else:
@@ -66,16 +64,16 @@ def _get_filesystem_and_path(passed_filesystem, path):
 
 
 def _check_contains_null(val):
-    if isinstance(val, six.binary_type):
+    if isinstance(val, bytes):
         for byte in val:
-            if isinstance(byte, six.binary_type):
+            if isinstance(byte, bytes):
                 compare_to = chr(0)
             else:
                 compare_to = 0
             if byte == compare_to:
                 return True
-    elif isinstance(val, six.text_type):
-        return u'\x00' in val
+    elif isinstance(val, str):
+        return '\x00' in val
     return False
 
 
@@ -86,7 +84,7 @@ def _check_filters(filters):
     if filters is not None:
         if len(filters) == 0 or any(len(f) == 0 for f in filters):
             raise ValueError("Malformed filters")
-        if isinstance(filters[0][0], six.string_types):
+        if isinstance(filters[0][0], str):
             # We have encountered the situation where we have one nesting level
             # too few:
             #   We have [(,,), ..] instead of [[(,,), ..]]
@@ -108,7 +106,7 @@ def _check_filters(filters):
 # Reading a single Parquet file
 
 
-class ParquetFile(object):
+class ParquetFile:
     """
     Reader interface for a single Parquet file
 
@@ -385,7 +383,7 @@ compression_level: int or dict, default None
     a compression level."""
 
 
-class ParquetWriter(object):
+class ParquetWriter:
 
     __doc__ = """
 Class for incrementally building a Parquet file for Arrow tables
@@ -394,7 +392,7 @@ Parameters
 ----------
 where : path or file-like object
 schema : arrow Schema
-{0}
+{}
 **options : dict
     If options contains a key `metadata_collector` then the
     corresponding value is assumed to be a list (or any object with
@@ -467,8 +465,8 @@ schema : arrow Schema
 
         if not table.schema.equals(self.schema, check_metadata=False):
             msg = ('Table schema does not match schema used to create file: '
-                   '\ntable:\n{0!s} vs. \nfile:\n{1!s}'.format(table.schema,
-                                                               self.schema))
+                   '\ntable:\n{!s} vs. \nfile:\n{!s}'
+                   .format(table.schema, self.schema))
             raise ValueError(msg)
 
         self.writer.write_table(table, row_group_size=row_group_size)
@@ -493,7 +491,7 @@ def _get_pandas_index_columns(keyvalues):
 # file, possibly part of a partitioned dataset
 
 
-class ParquetDatasetPiece(object):
+class ParquetDatasetPiece:
     """
     A single chunk of a potentially larger Parquet dataset to read. The
     arguments will indicate to read either a single row group or all row
@@ -525,11 +523,8 @@ class ParquetDatasetPiece(object):
                 self.row_group == other.row_group and
                 self.partition_keys == other.partition_keys)
 
-    def __ne__(self, other):
-        return not (self == other)
-
     def __repr__(self):
-        return ('{0}({1!r}, row_group={2!r}, partition_keys={3!r})'
+        return ('{}({!r}, row_group={!r}, partition_keys={!r})'
                 .format(type(self).__name__, self.path,
                         self.row_group,
                         self.partition_keys))
@@ -538,14 +533,14 @@ class ParquetDatasetPiece(object):
         result = ''
 
         if len(self.partition_keys) > 0:
-            partition_str = ', '.join('{0}={1}'.format(name, index)
+            partition_str = ', '.join('{}={}'.format(name, index)
                                       for name, index in self.partition_keys)
-            result += 'partition[{0}] '.format(partition_str)
+            result += 'partition[{}] '.format(partition_str)
 
         result += self.path
 
         if self.row_group is not None:
-            result += ' | row_group={0}'.format(self.row_group)
+            result += ' | row_group={}'.format(self.row_group)
 
         return result
 
@@ -633,7 +628,7 @@ class ParquetDatasetPiece(object):
         return table
 
 
-class PartitionSet(object):
+class PartitionSet:
     """A data structure for cataloguing the observed Parquet partitions at a
     particular level. So if we have
 
@@ -691,7 +686,7 @@ class PartitionSet(object):
         return list(self.keys) == sorted(self.keys)
 
 
-class ParquetPartitions(object):
+class ParquetPartitions:
 
     def __init__(self):
         self.levels = []
@@ -716,10 +711,6 @@ class ParquetPartitions(object):
         except TypeError:
             return NotImplemented
 
-    def __ne__(self, other):
-        # required for python 2, cython implements it by default
-        return not (self == other)
-
     def get_index(self, level, name, key):
         """
         Record a partition value at a particular level, returning the distinct
@@ -741,7 +732,7 @@ class ParquetPartitions(object):
         """
         if level == len(self.levels):
             if name in self.partition_names:
-                raise ValueError('{0} was the name of the partition in '
+                raise ValueError('{} was the name of the partition in '
                                  'another level'.format(name))
 
             part_set = PartitionSet(name)
@@ -764,14 +755,13 @@ class ParquetPartitions(object):
             if op not in {'in', 'not in'}:
                 raise ValueError("Op '%s' not supported with set value",
                                  op)
-            if len(set([type(item) for item in f_value])) != 1:
+            if len({type(item) for item in f_value}) != 1:
                 raise ValueError("All elements of set '%s' must be of"
                                  " same type", f_value)
             f_type = type(next(iter(f_value)))
 
-        p_value = f_type((self.levels[level]
-                          .dictionary[p_value_index]
-                          .as_py()))
+        p_value = f_type(self.levels[level]
+                         .dictionary[p_value_index].as_py())
 
         if op == "=" or op == "==":
             return p_value == f_value
@@ -794,7 +784,7 @@ class ParquetPartitions(object):
                              filter[1])
 
 
-class ParquetManifest(object):
+class ParquetManifest:
     """
 
     """
@@ -854,7 +844,7 @@ class ParquetManifest(object):
 
         if len(filtered_files) > 0 and len(filtered_directories) > 0:
             raise ValueError('Found files in an intermediate '
-                             'directory: {0}'.format(base_path))
+                             'directory: {}'.format(base_path))
         elif len(filtered_directories) > 0:
             self._visit_directories(level, filtered_directories, part_keys)
         else:
@@ -892,7 +882,7 @@ class ParquetManifest(object):
         if self.partition_scheme == 'hive':
             return _parse_hive_partition(dirname)
         else:
-            raise NotImplementedError('partition schema: {0}'
+            raise NotImplementedError('partition schema: {}'
                                       .format(self.partition_scheme))
 
     def _push_pieces(self, files, part_keys):
@@ -906,7 +896,7 @@ class ParquetManifest(object):
 def _parse_hive_partition(value):
     if '=' not in value:
         raise ValueError('Directory name did not appear to be a '
-                         'partition: {0}'.format(value))
+                         'partition: {}'.format(value))
     return value.split('=', 1)
 
 
@@ -959,7 +949,7 @@ buffer_size : int, default 0
     column chunks. Otherwise IO calls are unbuffered."""
 
 
-class ParquetDataset(object):
+class ParquetDataset:
 
     __doc__ = """
 Encapsulates details of reading a complete Parquet dataset possibly
@@ -1000,7 +990,7 @@ metadata_nthreads: int, default 1
     How many threads to allow the thread pool which is used to read the
     dataset metadata. Increasing this is helpful to read partitioned
     datasets.
-{0}
+{}
 """.format(_read_docstring_common)
 
     def __init__(self, path_or_paths, filesystem=None, schema=None,
@@ -1080,10 +1070,6 @@ metadata_nthreads: int, default 1
         except TypeError:
             return NotImplemented
 
-    def __ne__(self, other):
-        # required for python 2, cython implements it by default
-        return not (self == other)
-
     def validate_schemas(self):
         if self.metadata is None and self.schema is None:
             if self.common_metadata is not None:
@@ -1107,8 +1093,8 @@ metadata_nthreads: int, default 1
             file_metadata = piece.get_metadata()
             file_schema = file_metadata.schema.to_arrow_schema()
             if not dataset_schema.equals(file_schema, check_metadata=False):
-                raise ValueError('Schema in {0!s} was different. \n'
-                                 '{1!s}\n\nvs\n\n{2!s}'
+                raise ValueError('Schema in {!s} was different. \n'
+                                 '{!s}\n\nvs\n\n{!s}'
                                  .format(piece, file_schema,
                                          dataset_schema))
 
@@ -1225,7 +1211,7 @@ def _make_manifest(path_or_paths, fs, pathsep='/', metadata_nthreads=1,
         pieces = []
         for path in path_or_paths:
             if not fs.isfile(path):
-                raise IOError('Passed non-file path: {0}'
+                raise OSError('Passed non-file path: {}'
                               .format(path))
             piece = ParquetDatasetPiece(path, open_file_func=open_file_func)
             pieces.append(piece)
@@ -1360,7 +1346,7 @@ table : pyarrow.Table
 where: string or pyarrow.NativeFile
 row_group_size: int
     The number of rows per rowgroup
-{0}
+{}
 """.format(_parquet_writer_arg_docs)
 
 

python/pyarrow/plasma.py

@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
 
 import contextlib
 import os

python/pyarrow/serialization.pxi

@@ -17,8 +17,6 @@
 
 from cpython.ref cimport PyObject
 
-import six
-
 from pyarrow.compat import frombytes, pickle
 
 
@@ -30,7 +28,7 @@ def is_named_tuple(cls):
     f = getattr(cls, "_fields", None)
     if not isinstance(f, tuple):
         return False
-    return all(isinstance(n, six.string_types) for n in f)
+    return all(isinstance(n, str) for n in f)
 
 
 class SerializationCallbackError(ArrowSerializationError):
@@ -119,7 +117,7 @@ cdef class SerializationContext:
             This argument is optional, but can be provided to
             deserialize objects of the class in a particular way.
         """
-        if not isinstance(type_id, six.string_types):
+        if not isinstance(type_id, str):
             raise TypeError("The type_id argument must be a string. The value "
                             "passed in has type {}.".format(type(type_id)))
 

python/pyarrow/serialization.py

@@ -15,11 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
-
 import collections
-import six
-import sys
 
 import numpy as np
 
@@ -129,10 +125,7 @@ def _pickle_to_buffer(x):
 
 def _load_pickle_from_buffer(data):
     as_memoryview = memoryview(data)
-    if six.PY2:
-        return builtin_pickle.loads(as_memoryview.tobytes())
-    else:
-        return builtin_pickle.loads(as_memoryview)
+    return builtin_pickle.loads(as_memoryview)
 
 
 # ----------------------------------------------------------------------
@@ -394,12 +387,6 @@ def register_default_serialization_handlers(serialization_context):
         custom_serializer=lambda obj: str(obj),
         custom_deserializer=lambda data: int(data))
 
-    if (sys.version_info < (3, 0)):
-        serialization_context.register_type(
-            long, "long",  # noqa: F821
-            custom_serializer=lambda obj: str(obj),
-            custom_deserializer=lambda data: long(data))  # noqa: F821
-
     serialization_context.register_type(
         type(lambda: 0), "function",
         pickle=True)

python/pyarrow/table.pxi

@@ -136,7 +136,7 @@ cdef class ChunkedArray(_PandasConvertible):
     def __getitem__(self, key):
         if isinstance(key, slice):
             return _normalize_slice(self, key)
-        elif isinstance(key, six.integer_types):
+        elif isinstance(key, int):
             return self.getitem(key)
         else:
             raise TypeError("key must either be a slice or integer")
@@ -457,7 +457,7 @@ cdef _schema_from_arrays(arrays, names, metadata, shared_ptr[CSchema]* schema):
         c_type = (<DataType> val.type).sp_type
 
         if names[i] is None:
-            c_name = tobytes(u'None')
+            c_name = b'None'
         else:
             c_name = tobytes(names[i])
         c_fields[i].reset(new CField(c_name, c_type, True))
@@ -1429,13 +1429,13 @@ cdef class Table(_PandasConvertible):
         -------
         pyarrow.ChunkedArray
         """
-        if isinstance(i, six.string_types):
+        if isinstance(i, (bytes, str)):
             field_index = self.schema.get_field_index(i)
             if field_index < 0:
                 raise KeyError("Column {} does not exist in table".format(i))
             else:
                 return self._column(field_index)
-        elif isinstance(i, six.integer_types):
+        elif isinstance(i, int):
             return self._column(i)
         else:
             raise TypeError("Index must either be string or integer")

python/pyarrow/tests/conftest.py

@@ -16,17 +16,13 @@
 # under the License.
 
 import os
+import pathlib
 import subprocess
-import tempfile
+from tempfile import TemporaryDirectory
 
 import pytest
 import hypothesis as h
 
-try:
-    import pathlib
-except ImportError:
-    import pathlib2 as pathlib  # py2 compat
-
 from pyarrow.util import find_free_port
 
 
@@ -178,7 +174,7 @@ def pytest_addoption(parser):
                              help=('Enable the {} test group'.format(group)))
 
 
-class PyArrowConfig(object):
+class PyArrowConfig:
     def __init__(self):
         self.is_enabled = {}
 
@@ -189,7 +185,7 @@ class PyArrowConfig(object):
 
     def requires(self, group):
         if not self.is_enabled[group]:
-            pytest.skip('{0} NOT enabled'.format(group))
+            pytest.skip('{} NOT enabled'.format(group))
 
 
 def pytest_configure(config):
@@ -201,8 +197,8 @@ def pytest_configure(config):
             "markers", mark,
         )
 
-        flag = '--{0}'.format(mark)
-        enable_flag = '--enable-{0}'.format(mark)
+        flag = '--{}'.format(mark)
+        enable_flag = '--enable-{}'.format(mark)
 
         is_enabled = (config.getoption(flag) or
                       config.getoption(enable_flag))
@@ -226,22 +222,6 @@ def datadir():
     return pathlib.Path(__file__).parent / 'data'
 
 
-try:
-    from tempfile import TemporaryDirectory
-except ImportError:
-    import shutil
-
-    class TemporaryDirectory(object):
-        """Temporary directory implementation for python 2"""
-
-        def __enter__(self):
-            self.tmp = tempfile.mkdtemp()
-            return self.tmp
-
-        def __exit__(self, exc_type, exc_value, traceback):
-            shutil.rmtree(self.tmp)
-
-
 # TODO(kszucs): move the following fixtures to test_fs.py once the previous
 # parquet dataset implementation and hdfs implementation are removed.
 
@@ -273,7 +253,7 @@ def minio_server():
         proc = None
         try:
             proc = subprocess.Popen(args, env=env)
-        except (OSError, IOError):
+        except OSError:
             pytest.skip('`minio` command cannot be located')
         else:
             yield address, access_key, secret_key

python/pyarrow/tests/pandas_examples.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -53,10 +52,10 @@ def dataframe_with_arrays(include_index=False):
 
     fields.append(pa.field('str', pa.list_(pa.string())))
     arrays['str'] = [
-        np.array([u"1", u"ä"], dtype="object"),
+        np.array(["1", "ä"], dtype="object"),
         None,
-        np.array([u"1"], dtype="object"),
-        np.array([u"1", u"2", u"3"], dtype="object")
+        np.array(["1"], dtype="object"),
+        np.array(["1", "2", "3"], dtype="object")
     ]
 
     fields.append(pa.field('datetime64', pa.list_(pa.timestamp('ms'))))
@@ -123,10 +122,10 @@ def dataframe_with_lists(include_index=False, parquet_compatible=False):
     ]
     fields.append(pa.field('str_list', pa.list_(pa.string())))
     arrays['str_list'] = [
-        [u"1", u"ä"],
+        ["1", "ä"],
         None,
-        [u"1"],
-        [u"1", u"2", u"3"],
+        ["1"],
+        ["1", "2", "3"],
         [],
     ]
 

python/pyarrow/tests/test_array.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -72,7 +71,7 @@ def test_list_format():
 
 
 def test_string_format():
-    arr = pa.array([u'', None, u'foo'])
+    arr = pa.array(['', None, 'foo'])
     result = arr.format()
     expected = """\
 [
@@ -396,7 +395,7 @@ def test_array_from_buffers():
     assert arr.to_pylist() == [None, 6, 7]
 
     with pytest.raises(TypeError):
-        pa.Array.from_buffers(pa.int16(), 3, [u'', u''], offset=1)
+        pa.Array.from_buffers(pa.int16(), 3, ['', ''], offset=1)
 
 
 def test_string_binary_from_buffers():
@@ -1023,9 +1022,9 @@ def test_cast_from_null():
 
 def test_cast_string_to_number_roundtrip():
     cases = [
-        (pa.array([u"1", u"127", u"-128"]),
+        (pa.array(["1", "127", "-128"]),
          pa.array([1, 127, -128], type=pa.int8())),
-        (pa.array([None, u"18446744073709551615"]),
+        (pa.array([None, "18446744073709551615"]),
          pa.array([None, 18446744073709551615], type=pa.uint64())),
     ]
     for in_arr, expected in cases:
@@ -1040,8 +1039,8 @@ def test_cast_string_to_number_roundtrip():
 def test_cast_dictionary():
     arr = pa.DictionaryArray.from_arrays(
         pa.array([0, 1, None], type=pa.int32()),
-        pa.array([u"foo", u"bar"]))
-    assert arr.cast(pa.string()).equals(pa.array([u"foo", u"bar", None]))
+        pa.array(["foo", "bar"]))
+    assert arr.cast(pa.string()).equals(pa.array(["foo", "bar", None]))
     with pytest.raises(NotImplementedError):
         # Shouldn't crash (ARROW-7077)
         arr.cast(pa.int32())
@@ -1172,7 +1171,7 @@ def test_cast_binary_to_utf8():
 
     assert utf8_arr.equals(expected)
 
-    non_utf8_values = [(u'mañana').encode('utf-16-le')]
+    non_utf8_values = [('mañana').encode('utf-16-le')]
     non_utf8_binary = pa.array(non_utf8_values)
     assert non_utf8_binary.type == pa.binary()
     with pytest.raises(ValueError):
@@ -1994,7 +1993,7 @@ def test_array_from_numpy_str_utf8():
 
     arr = pa.array(vec, pa.string())
     arr2 = pa.array(vec2, pa.string())
-    expected = pa.array([u"toto", u"tata"])
+    expected = pa.array(["toto", "tata"])
     assert arr.equals(expected)
     assert arr2.equals(expected)
 
@@ -2004,7 +2003,7 @@ def test_array_from_numpy_str_utf8():
     assert arr.equals(expected)
 
     # UTF8 validation failures
-    vec = np.array([(u'mañana').encode('utf-16-le')])
+    vec = np.array([('mañana').encode('utf-16-le')])
     with pytest.raises(ValueError):
         pa.array(vec, pa.string())
 
@@ -2018,7 +2017,7 @@ def test_numpy_binary_overflow_to_chunked():
 
     # 2^31 + 1 bytes
     values = [b'x']
-    unicode_values = [u'x']
+    unicode_values = ['x']
 
     # Make 10 unique 1MB strings then repeat then 2048 times
     unique_strings = {
@@ -2060,12 +2059,12 @@ def test_list_child_overflow_to_chunked():
 
 def test_infer_type_masked():
     # ARROW-5208
-    ty = pa.infer_type([u'foo', u'bar', None, 2],
+    ty = pa.infer_type(['foo', 'bar', None, 2],
                        mask=[False, False, False, True])
     assert ty == pa.utf8()
 
     # all masked
-    ty = pa.infer_type([u'foo', u'bar', None, 2],
+    ty = pa.infer_type(['foo', 'bar', None, 2],
                        mask=np.array([True, True, True, True]))
     assert ty == pa.null()
 

python/pyarrow/tests/test_compute.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

python/pyarrow/tests/test_convert_builtin.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -18,7 +17,7 @@
 
 import pytest
 
-from pyarrow.compat import unittest, u  # noqa
+from pyarrow.compat import unittest  # noqa
 from pyarrow.pandas_compat import _pandas_api  # noqa
 import pyarrow as pa
 
@@ -28,11 +27,9 @@ import decimal
 import itertools
 import math
 import traceback
-import sys
 
 import numpy as np
 import pytz
-import six
 
 
 int_type_pairs = [
@@ -145,7 +142,7 @@ def _as_deque(xs):
 def _as_dict_values(xs):
     # a dict values object is not a sequence, just a regular iterable
     dct = {k: v for k, v in enumerate(xs)}
-    return six.viewvalues(dct)
+    return dct.values()
 
 
 def _as_numpy_array(xs):
@@ -651,7 +648,7 @@ def test_nested_ndarray_different_dtypes():
 
 
 def test_sequence_unicode():
-    data = [u'foo', u'bar', None, u'mañana']
+    data = ['foo', 'bar', None, 'mañana']
     arr = pa.array(data)
     assert len(arr) == 4
     assert arr.null_count == 1
@@ -660,9 +657,9 @@ def test_sequence_unicode():
 
 
 def check_array_mixed_unicode_bytes(binary_type, string_type):
-    values = [u'qux', b'foo', bytearray(b'barz')]
+    values = ['qux', b'foo', bytearray(b'barz')]
     b_values = [b'qux', b'foo', b'barz']
-    u_values = [u'qux', u'foo', u'barz']
+    u_values = ['qux', 'foo', 'barz']
 
     arr = pa.array(values)
     expected = pa.array(b_values, type=pa.binary())
@@ -732,10 +729,10 @@ def test_sequence_utf8_to_unicode(ty):
     data = [b'foo', None, b'bar']
     arr = pa.array(data, type=ty)
     assert arr.type == ty
-    assert arr[0].as_py() == u'foo'
+    assert arr[0].as_py() == 'foo'
 
     # test a non-utf8 unicode string
-    val = (u'mañana').encode('utf-16-le')
+    val = ('mañana').encode('utf-16-le')
     with pytest.raises(pa.ArrowInvalid):
         pa.array([val], type=ty)
 
@@ -990,12 +987,7 @@ def test_sequence_timestamp_from_int_with_unit():
     assert repr(arr_ns[0]) == "Timestamp('1970-01-01 00:00:00.000000001')"
     assert str(arr_ns[0]) == "1970-01-01 00:00:00.000000001"
 
-    if sys.version_info >= (3,):
-        expected_exc = TypeError
-    else:
-        # Can have "AttributeError: CustomClass instance
-        # has no attribute '__trunc__'"
-        expected_exc = (TypeError, AttributeError)
+    expected_exc = TypeError
 
     class CustomClass():
         pass
@@ -1218,16 +1210,16 @@ def test_structarray():
     assert arr.to_pylist() == []
 
     ints = pa.array([None, 2, 3], type=pa.int64())
-    strs = pa.array([u'a', None, u'c'], type=pa.string())
+    strs = pa.array(['a', None, 'c'], type=pa.string())
     bools = pa.array([True, False, None], type=pa.bool_())
     arr = pa.StructArray.from_arrays(
         [ints, strs, bools],
         ['ints', 'strs', 'bools'])
 
     expected = [
-        {'ints': None, 'strs': u'a', 'bools': True},
+        {'ints': None, 'strs': 'a', 'bools': True},
         {'ints': 2, 'strs': None, 'bools': False},
-        {'ints': 3, 'strs': u'c', 'bools': None},
+        {'ints': 3, 'strs': 'c', 'bools': None},
     ]
 
     pylist = arr.to_pylist()
@@ -1329,8 +1321,8 @@ def test_struct_from_dicts_inference():
     expected_type = pa.struct([pa.field('a', pa.int64()),
                                pa.field('b', pa.string()),
                                pa.field('c', pa.bool_())])
-    data = [{'a': 5, 'b': u'foo', 'c': True},
-            {'a': 6, 'b': u'bar', 'c': False}]
+    data = [{'a': 5, 'b': 'foo', 'c': True},
+            {'a': 6, 'b': 'bar', 'c': False}]
     arr = pa.array(data)
     check_struct_type(arr.type, expected_type)
     assert arr.to_pylist() == data
@@ -1339,11 +1331,11 @@ def test_struct_from_dicts_inference():
     data = [{'a': 5, 'c': True},
             None,
             {},
-            {'a': None, 'b': u'bar'}]
+            {'a': None, 'b': 'bar'}]
     expected = [{'a': 5, 'b': None, 'c': True},
                 None,
                 {'a': None, 'b': None, 'c': None},
-                {'a': None, 'b': u'bar', 'c': None}]
+                {'a': None, 'b': 'bar', 'c': None}]
     arr = pa.array(data)
     data_as_ndarray = np.empty(len(data), dtype=object)
     data_as_ndarray[:] = data
@@ -1377,7 +1369,7 @@ def test_struct_from_dicts_inference():
 def test_structarray_from_arrays_coerce():
     # ARROW-1706
     ints = [None, 2, 3]
-    strs = [u'a', None, u'c']
+    strs = ['a', None, 'c']
     bools = [True, False, None]
     ints_nonnull = [1, 2, 3]
 

python/pyarrow/tests/test_csv.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -41,21 +39,20 @@ from pyarrow.csv import read_csv, ReadOptions, ParseOptions, ConvertOptions
 def generate_col_names():
     # 'a', 'b'... 'z', then 'aa', 'ab'...
     letters = string.ascii_lowercase
-    for letter in letters:
-        yield letter
-    for first in letter:
-        for second in letter:
+    yield from letters
+    for first in letters:
+        for second in letters:
             yield first + second
 
 
-def make_random_csv(num_cols=2, num_rows=10, linesep=u'\r\n'):
+def make_random_csv(num_cols=2, num_rows=10, linesep='\r\n'):
     arr = np.random.RandomState(42).randint(0, 1000, size=(num_cols, num_rows))
     col_names = list(itertools.islice(generate_col_names(), num_cols))
     csv = io.StringIO()
-    csv.write(u",".join(col_names))
+    csv.write(",".join(col_names))
     csv.write(linesep)
     for row in arr.T:
-        csv.write(u",".join(map(str, row)))
+        csv.write(",".join(map(str, row)))
         csv.write(linesep)
     csv = csv.getvalue().encode()
     columns = [pa.array(a, type=pa.int64()) for a in arr]
@@ -65,8 +62,8 @@ def make_random_csv(num_cols=2, num_rows=10, linesep=u'\r\n'):
 
 def make_empty_csv(column_names):
     csv = io.StringIO()
-    csv.write(u",".join(column_names))
-    csv.write(u"\n")
+    csv.write(",".join(column_names))
+    csv.write("\n")
     return csv.getvalue().encode()
 
 
@@ -436,7 +433,7 @@ class BaseTestCSVRead:
         assert table.to_pydict() == {
             'a': [1.0, 4.0],
             'b': [2, -5],
-            'c': [u"3", u"foo"],
+            'c': ["3", "foo"],
             'd': [False, True],
             }
 
@@ -457,7 +454,7 @@ class BaseTestCSVRead:
         assert table.to_pydict() == {
             'a': [1.0, None, 4.5],
             'b': [2, -5, None],
-            'c': [u"", u"foo", u"nan"],
+            'c': ["", "foo", "nan"],
             'd': [None, None, None],
             'e': [b"3", b"nan", b"\xff"],
             'f': [None, True, False],
@@ -477,12 +474,12 @@ class BaseTestCSVRead:
 
     def test_auto_dict_encode(self):
         opts = ConvertOptions(auto_dict_encode=True)
-        rows = u"a,b\nab,1\ncdé,2\ncdé,3\nab,4".encode('utf8')
+        rows = "a,b\nab,1\ncdé,2\ncdé,3\nab,4".encode()
         table = self.read_bytes(rows, convert_options=opts)
         schema = pa.schema([('a', pa.dictionary(pa.int32(), pa.string())),
                             ('b', pa.int64())])
         expected = {
-            'a': [u"ab", u"cdé", u"cdé", u"ab"],
+            'a': ["ab", "cdé", "cdé", "ab"],
             'b': [1, 2, 3, 4],
             }
         assert table.schema == schema
@@ -508,7 +505,7 @@ class BaseTestCSVRead:
         assert table.schema == schema
         dict_values = table['a'].chunk(0).dictionary
         assert len(dict_values) == 2
-        assert dict_values[0] == u"ab"
+        assert dict_values[0] == "ab"
         assert dict_values[1].as_buffer() == b"cd\xff"
 
         # With invalid UTF8, checked
@@ -535,8 +532,8 @@ class BaseTestCSVRead:
         assert table.schema == schema
         assert table.to_pydict() == {
             'a': [None, None],
-            'b': [u"Xxx", u"#N/A"],
-            'c': [u"1", u""],
+            'b': ["Xxx", "#N/A"],
+            'c': ["1", ""],
             'd': [2, None],
             }
 
@@ -545,8 +542,8 @@ class BaseTestCSVRead:
         table = self.read_bytes(rows, convert_options=opts)
         assert table.to_pydict() == {
             'a': [None, None],
-            'b': [None, u"#N/A"],
-            'c': [u"1", u""],
+            'b': [None, "#N/A"],
+            'c': ["1", ""],
             'd': [2, None],
             }
 
@@ -557,8 +554,8 @@ class BaseTestCSVRead:
                             ('b', pa.string())])
         assert table.schema == schema
         assert table.to_pydict() == {
-            'a': [u"#N/A"],
-            'b': [u""],
+            'a': ["#N/A"],
+            'b': [""],
             }
 
     def test_custom_bools(self):
@@ -692,14 +689,14 @@ class BaseTestCSVRead:
         rows = b"a;b,c\nde,fg;eh\n"
         table = self.read_bytes(rows)
         assert table.to_pydict() == {
-            'a;b': [u'de'],
-            'c': [u'fg;eh'],
+            'a;b': ['de'],
+            'c': ['fg;eh'],
             }
         opts = ParseOptions(delimiter=';')
         table = self.read_bytes(rows, parse_options=opts)
         assert table.to_pydict() == {
-            'a': [u'de,fg'],
-            'b,c': [u'eh'],
+            'a': ['de,fg'],
+            'b,c': ['eh'],
             }
 
     def test_small_random_csv(self):
@@ -731,7 +728,7 @@ class BaseTestCSVRead:
         except AttributeError:
             clock = time.time
         num_columns = 10000
-        col_names = ["K{0}".format(i) for i in range(num_columns)]
+        col_names = ["K{}".format(i) for i in range(num_columns)]
         csv = make_empty_csv(col_names)
         t1 = clock()
         convert_options = ConvertOptions(

python/pyarrow/tests/test_cuda.py

@@ -685,7 +685,6 @@ def other_process_for_test_IPC(handle_buffer, expected_arr):
 
 
 @cuda_ipc
-@pytest.mark.skipif(sys.version_info[0] == 2, reason="test needs Python 3")
 @pytest.mark.parametrize("size", [0, 1, 1000])
 def test_IPC(size):
     import multiprocessing

python/pyarrow/tests/test_dataset.py

@@ -617,7 +617,7 @@ def test_open_dataset_partitioned_directory(tempdir):
     import pyarrow.parquet as pq
     table = pa.table({'a': range(9), 'b': [0.] * 4 + [1.] * 5})
     for part in range(3):
-        path = tempdir / "part={0}".format(part)
+        path = tempdir / "part={}".format(part)
         path.mkdir()
         pq.write_table(table, path / "test.parquet")
 

python/pyarrow/tests/test_feather.py

@@ -356,14 +356,14 @@ class TestFeatherReader(unittest.TestCase):
         if sys.platform == 'win32':
             pytest.skip('Windows hangs on to file handle for some reason')
 
-        class CustomClass(object):
+        class CustomClass:
             pass
 
         # strings will fail
         df = pd.DataFrame(
             {
                 'numbers': range(5),
-                'strings': [b'foo', None, u'bar', CustomClass(), np.nan]},
+                'strings': [b'foo', None, 'bar', CustomClass(), np.nan]},
             columns=['numbers', 'strings'])
 
         path = random_path()
@@ -378,7 +378,7 @@ class TestFeatherReader(unittest.TestCase):
         repeats = 1000
 
         # Mixed bytes, unicode, strings coerced to binary
-        values = [b'foo', None, u'bar', 'qux', np.nan]
+        values = [b'foo', None, 'bar', 'qux', np.nan]
         df = pd.DataFrame({'strings': values * repeats})
 
         ex_values = [b'foo', None, b'bar', b'qux', np.nan]
@@ -411,7 +411,7 @@ class TestFeatherReader(unittest.TestCase):
         self._check_pandas_roundtrip(df, null_counts=[0, 3])
 
     def test_multithreaded_read(self):
-        data = {'c{0}'.format(i): [''] * 10
+        data = {'c{}'.format(i): [''] * 10
                 for i in range(100)}
         df = pd.DataFrame(data)
         self._check_pandas_roundtrip(df, use_threads=True)
@@ -424,7 +424,7 @@ class TestFeatherReader(unittest.TestCase):
 
     def test_category(self):
         repeats = 1000
-        values = ['foo', None, u'bar', 'qux', np.nan]
+        values = ['foo', None, 'bar', 'qux', np.nan]
         df = pd.DataFrame({'strings': values * repeats})
         df['strings'] = df['strings'].astype('category')
 

python/pyarrow/tests/test_flight.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -122,7 +121,7 @@ class ConstantFlightServer(FlightServerBase):
     """
 
     def __init__(self, location=None, **kwargs):
-        super(ConstantFlightServer, self).__init__(location, **kwargs)
+        super().__init__(location, **kwargs)
         # Ticket -> Table
         self.table_factories = {
             b'ints': simple_ints_table,
@@ -177,7 +176,7 @@ class EchoFlightServer(FlightServerBase):
     """A Flight server that returns the last data uploaded."""
 
     def __init__(self, location=None, expected_schema=None, **kwargs):
-        super(EchoFlightServer, self).__init__(location, **kwargs)
+        super().__init__(location, **kwargs)
         self.last_message = None
         self.expected_schema = expected_schema
 
@@ -242,8 +241,7 @@ class ListActionsFlightServer(FlightServerBase):
         ]
 
     def list_actions(self, context):
-        for action in self.expected_actions():
-            yield action
+        yield from self.expected_actions()
 
 
 class ListActionsErrorFlightServer(FlightServerBase):
@@ -258,7 +256,7 @@ class CheckTicketFlightServer(FlightServerBase):
     """A Flight server that compares the given ticket to an expected value."""
 
     def __init__(self, expected_ticket, location=None, **kwargs):
-        super(CheckTicketFlightServer, self).__init__(location, **kwargs)
+        super().__init__(location, **kwargs)
         self.expected_ticket = expected_ticket
 
     def do_get(self, context, ticket):
@@ -338,7 +336,7 @@ class HttpBasicServerAuthHandler(ServerAuthHandler):
     """An example implementation of HTTP basic authentication."""
 
     def __init__(self, creds):
-        super(HttpBasicServerAuthHandler, self).__init__()
+        super().__init__()
         self.creds = creds
 
     def authenticate(self, outgoing, incoming):
@@ -362,7 +360,7 @@ class HttpBasicClientAuthHandler(ClientAuthHandler):
     """An example implementation of HTTP basic authentication."""
 
     def __init__(self, username, password):
-        super(HttpBasicClientAuthHandler, self).__init__()
+        super().__init__()
         self.basic_auth = flight.BasicAuth(username, password)
         self.token = None
 
@@ -379,7 +377,7 @@ class TokenServerAuthHandler(ServerAuthHandler):
     """An example implementation of authentication via handshake."""
 
     def __init__(self, creds):
-        super(TokenServerAuthHandler, self).__init__()
+        super().__init__()
         self.creds = creds
 
     def authenticate(self, outgoing, incoming):
@@ -402,7 +400,7 @@ class TokenClientAuthHandler(ClientAuthHandler):
     """An example implementation of authentication via handshake."""
 
     def __init__(self, username, password):
-        super(TokenClientAuthHandler, self).__init__()
+        super().__init__()
         self.username = username
         self.password = password
         self.token = b''
@@ -437,7 +435,7 @@ class HeaderFlightServer(FlightServerBase):
         middleware = context.get_middleware("test")
         if middleware:
             return iter([flight.Result(middleware.special_value.encode())])
-        return iter([flight.Result("".encode())])
+        return iter([flight.Result(b"")])
 
 
 class SelectiveAuthServerMiddlewareFactory(ServerMiddlewareFactory):

python/pyarrow/tests/test_fs.py

@@ -16,16 +16,13 @@
 # under the License.
 
 from datetime import datetime
-try:
-    import pathlib
-except ImportError:
-    import pathlib2 as pathlib  # py2 compat
+import gzip
+import pathlib
 
 import pytest
 
 import pyarrow as pa
-from pyarrow.tests.test_io import (gzip_compress, gzip_decompress,
-                                   assert_file_not_found)
+from pyarrow.tests.test_io import assert_file_not_found
 from pyarrow.fs import (FileType, FileSelector, FileSystem, LocalFileSystem,
                         LocalFileSystemOptions, SubTreeFileSystem,
                         _MockFileSystem)
@@ -387,8 +384,8 @@ def identity(v):
     [
         (None, None, identity),
         (None, 64, identity),
-        ('gzip', None, gzip_compress),
-        ('gzip', 256, gzip_compress),
+        ('gzip', None, gzip.compress),
+        ('gzip', 256, gzip.compress),
     ]
 )
 def test_open_input_stream(fs, pathfn, compression, buffer_size, compressor):
@@ -424,8 +421,8 @@ def test_open_input_file(fs, pathfn):
     [
         (None, None, identity),
         (None, 64, identity),
-        ('gzip', None, gzip_decompress),
-        ('gzip', 256, gzip_decompress),
+        ('gzip', None, gzip.decompress),
+        ('gzip', 256, gzip.decompress),
     ]
 )
 def test_open_output_stream(fs, pathfn, compression, buffer_size,
@@ -445,8 +442,8 @@ def test_open_output_stream(fs, pathfn, compression, buffer_size,
     [
         (None, None, identity, identity),
         (None, 64, identity, identity),
-        ('gzip', None, gzip_compress, gzip_decompress),
-        ('gzip', 256, gzip_compress, gzip_decompress),
+        ('gzip', None, gzip.compress, gzip.decompress),
+        ('gzip', 256, gzip.compress, gzip.decompress),
     ]
 )
 def test_open_append_stream(fs, pathfn, compression, buffer_size, compressor,

python/pyarrow/tests/test_gandiva.py

@@ -104,13 +104,13 @@ def test_filter():
 def test_in_expr():
     import pyarrow.gandiva as gandiva
 
-    arr = pa.array([u"ga", u"an", u"nd", u"di", u"iv", u"va"])
+    arr = pa.array(["ga", "an", "nd", "di", "iv", "va"])
     table = pa.Table.from_arrays([arr], ["a"])
 
     # string
     builder = gandiva.TreeExprBuilder()
     node_a = builder.make_field(table.schema.field("a"))
-    cond = builder.make_in_expression(node_a, [u"an", u"nd"], pa.string())
+    cond = builder.make_in_expression(node_a, ["an", "nd"], pa.string())
     condition = builder.make_condition(cond)
     filter = gandiva.make_filter(table.schema, condition)
     result = filter.evaluate(table.to_batches()[0], pa.default_memory_pool())

python/pyarrow/tests/test_hdfs.py

@@ -50,7 +50,7 @@ def hdfs_test_client(driver='libhdfs'):
 
 
 @pytest.mark.hdfs
-class HdfsTestCases(object):
+class HdfsTestCases:
 
     def _make_test_file(self, hdfs, test_name, test_path, test_data):
         base_path = pjoin(self.tmp_path, test_name)
@@ -67,7 +67,7 @@ class HdfsTestCases(object):
     def setUpClass(cls):
         cls.check_driver()
         cls.hdfs = hdfs_test_client(cls.DRIVER)
-        cls.tmp_path = '/tmp/pyarrow-test-{0}'.format(random.randint(0, 1000))
+        cls.tmp_path = '/tmp/pyarrow-test-{}'.format(random.randint(0, 1000))
         cls.hdfs.mkdir(cls.tmp_path)
 
     @classmethod
@@ -287,7 +287,7 @@ class HdfsTestCases(object):
             # Hack so that we don't have a dtype cast in v1 files
             df['uint32'] = df['uint32'].astype(np.int64)
 
-            path = pjoin(tmpdir, '{0}.parquet'.format(i))
+            path = pjoin(tmpdir, '{}.parquet'.format(i))
 
             table = pa.Table.from_pandas(df, preserve_index=False)
             with self.hdfs.open(path, 'wb') as f:

python/pyarrow/tests/test_io.py

@@ -21,20 +21,16 @@ from io import (BytesIO, StringIO, TextIOWrapper, BufferedIOBase, IOBase)
 import gc
 import gzip
 import os
+import pathlib
 import pickle
 import pytest
 import sys
 import tempfile
 import weakref
 
-try:
-    import pathlib
-except ImportError:
-    import pathlib2 as pathlib
-
 import numpy as np
 
-from pyarrow.compat import u, guid, PY2
+from pyarrow.compat import guid
 import pyarrow as pa
 
 
@@ -58,12 +54,8 @@ def check_large_seeks(file_factory):
 
 @contextmanager
 def assert_file_not_found():
-    if PY2:
-        with pytest.raises(IOError):
-            yield
-    else:
-        with pytest.raises(FileNotFoundError):
-            yield
+    with pytest.raises(FileNotFoundError):
+        yield
 
 
 # ----------------------------------------------------------------------
@@ -225,7 +217,7 @@ def test_bytes_reader():
 
 def test_bytes_reader_non_bytes():
     with pytest.raises(TypeError):
-        pa.BufferReader(u('some sample data'))
+        pa.BufferReader('some sample data')
 
 
 def test_bytes_reader_retains_parent_reference():
@@ -442,7 +434,7 @@ def test_buffer_eq_bytes():
     assert buf != b'some dat1'
 
     with pytest.raises(TypeError):
-        buf == u'some data'
+        buf == 'some data'
 
 
 def test_buffer_getitem():
@@ -902,7 +894,7 @@ def test_native_file_write_reject_unicode():
     # ARROW-3227
     nf = pa.BufferOutputStream()
     with pytest.raises(TypeError):
-        nf.write(u'foo')
+        nf.write('foo')
 
 
 def test_native_file_modes(tmpdir):
@@ -987,9 +979,9 @@ def test_native_file_raises_ValueError_after_close(tmpdir):
 
 
 def test_native_file_TextIOWrapper(tmpdir):
-    data = (u'foooo\n'
-            u'barrr\n'
-            u'bazzz\n')
+    data = ('foooo\n'
+            'barrr\n'
+            'bazzz\n')
 
     path = os.path.join(str(tmpdir), guid())
     with open(path, 'wb') as f:
@@ -1249,40 +1241,13 @@ def test_compressed_recordbatch_stream(compression):
 # ----------------------------------------------------------------------
 # High-level API
 
-if PY2:
-    def gzip_compress(data):
-        fd, fn = tempfile.mkstemp(suffix='.gz')
-        try:
-            os.close(fd)
-            with gzip.GzipFile(fn, 'wb') as f:
-                f.write(data)
-            with open(fn, 'rb') as f:
-                return f.read()
-        finally:
-            os.unlink(fn)
-
-    def gzip_decompress(data):
-        fd, fn = tempfile.mkstemp(suffix='.gz')
-        try:
-            os.close(fd)
-            with open(fn, 'wb') as f:
-                f.write(data)
-            with gzip.GzipFile(fn, 'rb') as f:
-                return f.read()
-        finally:
-            os.unlink(fn)
-else:
-    gzip_compress = gzip.compress
-    gzip_decompress = gzip.decompress
-
-
 def test_input_stream_buffer():
     data = b"some test data\n" * 10 + b"eof\n"
     for arg in [pa.py_buffer(data), memoryview(data)]:
         stream = pa.input_stream(arg)
         assert stream.read() == data
 
-    gz_data = gzip_compress(data)
+    gz_data = gzip.compress(data)
     stream = pa.input_stream(memoryview(gz_data))
     assert stream.read() == gz_data
     stream = pa.input_stream(memoryview(gz_data), compression='gzip')
@@ -1291,7 +1256,7 @@ def test_input_stream_buffer():
 
 def test_input_stream_duck_typing():
     # Accept objects having the right file-like methods...
-    class DuckReader(object):
+    class DuckReader:
 
         def close(self):
             pass
@@ -1323,7 +1288,7 @@ def test_input_stream_file_path(tmpdir):
 
 def test_input_stream_file_path_compressed(tmpdir):
     data = b"some test data\n" * 10 + b"eof\n"
-    gz_data = gzip_compress(data)
+    gz_data = gzip.compress(data)
     file_path = tmpdir / 'input_stream.gz'
     with open(str(file_path), 'wb') as f:
         f.write(gz_data)
@@ -1369,7 +1334,7 @@ def test_input_stream_file_path_buffered(tmpdir):
 
 def test_input_stream_file_path_compressed_and_buffered(tmpdir):
     data = b"some test data\n" * 100 + b"eof\n"
-    gz_data = gzip_compress(data)
+    gz_data = gzip.compress(data)
     file_path = tmpdir / 'input_stream_compressed_and_buffered.gz'
     with open(str(file_path), 'wb') as f:
         f.write(gz_data)
@@ -1389,7 +1354,7 @@ def test_input_stream_python_file(tmpdir):
     stream = pa.input_stream(bio)
     assert stream.read() == data
 
-    gz_data = gzip_compress(data)
+    gz_data = gzip.compress(data)
     bio = BytesIO(gz_data)
     stream = pa.input_stream(bio)
     assert stream.read() == gz_data
@@ -1407,7 +1372,7 @@ def test_input_stream_python_file(tmpdir):
 
 def test_input_stream_native_file():
     data = b"some test data\n" * 10 + b"eof\n"
-    gz_data = gzip_compress(data)
+    gz_data = gzip.compress(data)
     reader = pa.BufferReader(gz_data)
     stream = pa.input_stream(reader)
     assert stream is reader
@@ -1448,7 +1413,7 @@ def test_output_stream_buffer():
 
 def test_output_stream_duck_typing():
     # Accept objects having the right file-like methods...
-    class DuckWriter(object):
+    class DuckWriter:
         def __init__(self):
             self.buf = pa.BufferOutputStream()
 
@@ -1493,12 +1458,12 @@ def test_output_stream_file_path_compressed(tmpdir):
         with open(str(file_path), 'rb') as f:
             return f.read()
 
-    assert gzip_decompress(check_data(file_path, data)) == data
-    assert gzip_decompress(check_data(str(file_path), data)) == data
-    assert gzip_decompress(
+    assert gzip.decompress(check_data(file_path, data)) == data
+    assert gzip.decompress(check_data(str(file_path), data)) == data
+    assert gzip.decompress(
         check_data(pathlib.Path(str(file_path)), data)) == data
 
-    assert gzip_decompress(
+    assert gzip.decompress(
         check_data(file_path, data, compression='gzip')) == data
     assert check_data(file_path, data, compression=None) == data
 
@@ -1544,13 +1509,13 @@ def test_output_stream_file_path_compressed_and_buffered(tmpdir):
             return f.read()
 
     result = check_data(file_path, data, buffer_size=32)
-    assert gzip_decompress(result) == data
+    assert gzip.decompress(result) == data
 
     result = check_data(file_path, data, buffer_size=1024)
-    assert gzip_decompress(result) == data
+    assert gzip.decompress(result) == data
 
     result = check_data(file_path, data, buffer_size=1024, compression='gzip')
-    assert gzip_decompress(result) == data
+    assert gzip.decompress(result) == data
 
 
 def test_output_stream_destructor(tmpdir):
@@ -1587,7 +1552,7 @@ def test_output_stream_python_file(tmpdir):
             return f.read()
 
     assert check_data(data) == data
-    assert gzip_decompress(check_data(data, compression='gzip')) == data
+    assert gzip.decompress(check_data(data, compression='gzip')) == data
 
 
 def test_output_stream_errors(tmpdir):

python/pyarrow/tests/test_ipc.py

@@ -38,7 +38,7 @@ except ImportError:
 pytestmark = pytest.mark.pandas
 
 
-class IpcFixture(object):
+class IpcFixture:
 
     def __init__(self, sink_factory=lambda: io.BytesIO()):
         self._sink_factory = sink_factory

python/pyarrow/tests/test_json.py

@@ -32,14 +32,13 @@ from pyarrow.json import read_json, ReadOptions, ParseOptions
 def generate_col_names():
     # 'a', 'b'... 'z', then 'aa', 'ab'...
     letters = string.ascii_lowercase
-    for letter in letters:
-        yield letter
-    for first in letter:
-        for second in letter:
+    yield from letters
+    for first in letters:
+        for second in letters:
             yield first + second
 
 
-def make_random_json(num_cols=2, num_rows=10, linesep=u'\r\n'):
+def make_random_json(num_cols=2, num_rows=10, linesep='\r\n'):
     arr = np.random.RandomState(42).randint(0, 1000, size=(num_cols, num_rows))
     col_names = list(itertools.islice(generate_col_names(), num_cols))
     lines = []
@@ -161,7 +160,7 @@ class BaseTestJSONRead:
         assert table.to_pydict() == {
             'a': [1.0, 4.0],
             'b': [2, -5],
-            'c': [u"3", u"foo"],
+            'c': ["3", "foo"],
             'd': [False, True],
             }
 
@@ -180,7 +179,7 @@ class BaseTestJSONRead:
         assert table.to_pydict() == {
             'a': [1.0, None, 4.5],
             'b': [2, -5, None],
-            'c': [None, u"foo", u"nan"],
+            'c': [None, "foo", "nan"],
             'd': [None, None, None],
             'e': [None, True, False],
             }

python/pyarrow/tests/test_jvm.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -21,7 +20,6 @@ import os
 import pyarrow as pa
 import pyarrow.jvm as pa_jvm
 import pytest
-import six
 import sys
 import xml.etree.ElementTree as ET
 
@@ -76,7 +74,7 @@ def _jvm_schema(jvm_spec, metadata=None):
     fields.add(field)
     if metadata:
         dct = jpype.JClass('java.util.HashMap')()
-        for k, v in six.iteritems(metadata):
+        for k, v in metadata.items():
             dct.put(k, v)
         return schema_cls(fields, dct)
     else:
@@ -372,7 +370,7 @@ def _string_to_varchar_holder(ra, string):
 @pytest.mark.xfail(reason="from_buffers is only supported for "
                           "primitive arrays yet")
 def test_jvm_string_array(root_allocator):
-    data = [u"string", None, u"töst"]
+    data = ["string", None, "töst"]
     cls = "org.apache.arrow.vector.VarCharVector"
     jvm_vector = jpype.JClass(cls)("vector", root_allocator)
     jvm_vector.allocateNew()

python/pyarrow/tests/test_orc.py

@@ -47,11 +47,7 @@ def fix_example_values(actual_cols, expected_cols):
         expected = expected_cols[name]
         actual = actual_cols[name]
         typ = actual[0].__class__
-        if typ is bytes:
-            # bytes fields are represented as lists of ints in JSON files
-            # (Python 2: need to use bytearray, not bytes)
-            expected = [bytearray(v) for v in expected]
-        elif issubclass(typ, datetime.datetime):
+        if issubclass(typ, datetime.datetime):
             # timestamp fields are represented as strings in JSON files
             expected = pd.to_datetime(expected)
         elif issubclass(typ, datetime.date):

python/pyarrow/tests/test_pandas.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -17,7 +16,6 @@
 # under the License.
 
 import gc
-import six
 import decimal
 import json
 import multiprocessing as mp
@@ -138,7 +136,7 @@ def _check_array_from_pandas_roundtrip(np_array, type=None):
     npt.assert_array_equal(result, np_array)
 
 
-class TestConvertMetadata(object):
+class TestConvertMetadata:
     """
     Conversion tests for Pandas metadata & indices.
     """
@@ -249,7 +247,7 @@ class TestConvertMetadata(object):
         tm.assert_frame_equal(df_expected, reconstructed)
 
     def test_multiindex_columns_unicode(self):
-        columns = pd.MultiIndex.from_arrays([[u'あ', u'い'], ['X', 'Y']])
+        columns = pd.MultiIndex.from_arrays([['あ', 'い'], ['X', 'Y']])
         df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')], columns=columns)
         _check_pandas_roundtrip(df, preserve_index=True)
 
@@ -330,17 +328,12 @@ class TestConvertMetadata(object):
         assert column_indexes['name'] == 'stringz'
         assert column_indexes['name'] == column_indexes['field_name']
         assert column_indexes['numpy_type'] == 'object'
-        assert column_indexes['pandas_type'] == (
-            'bytes' if six.PY2 else 'unicode'
-        )
+        assert column_indexes['pandas_type'] == 'unicode'
 
         md = column_indexes['metadata']
 
-        if not six.PY2:
-            assert len(md) == 1
-            assert md['encoding'] == 'UTF-8'
-        else:
-            assert md is None or 'encoding' not in md
+        assert len(md) == 1
+        assert md['encoding'] == 'UTF-8'
 
     def test_datetimetz_column_index(self):
         df = pd.DataFrame(
@@ -398,7 +391,7 @@ class TestConvertMetadata(object):
                 table.to_pandas()
 
     def test_unicode_with_unicode_column_and_index(self):
-        df = pd.DataFrame({u'あ': [u'い']}, index=[u'う'])
+        df = pd.DataFrame({'あ': ['い']}, index=['う'])
 
         _check_pandas_roundtrip(df, preserve_index=True)
 
@@ -406,7 +399,7 @@ class TestConvertMetadata(object):
         # mixed type column names are not reconstructed exactly
         df = pd.DataFrame({'a': [1, 2], 'b': [3, 4]})
 
-        for cols in [[u'あ', b'a'], [1, '2'], [1, 1.5]]:
+        for cols in [['あ', b'a'], [1, '2'], [1, 1.5]]:
             df.columns = pd.Index(cols, dtype=object)
 
             # assert that the from_pandas raises the warning
@@ -414,14 +407,14 @@ class TestConvertMetadata(object):
                 pa.Table.from_pandas(df)
 
             expected = df.copy()
-            expected.columns = df.columns.astype(six.text_type)
+            expected.columns = df.columns.astype(str)
             with pytest.warns(UserWarning):
                 _check_pandas_roundtrip(df, expected=expected,
                                         preserve_index=True)
 
     def test_binary_column_name(self):
-        column_data = [u'い']
-        key = u'あ'.encode('utf8')
+        column_data = ['い']
+        key = 'あ'.encode()
         data = {key: column_data}
         df = pd.DataFrame(data)
 
@@ -448,7 +441,7 @@ class TestConvertMetadata(object):
         tm.assert_frame_equal(result_df, df)
 
     def test_metadata_with_mixed_types(self):
-        df = pd.DataFrame({'data': [b'some_bytes', u'some_unicode']})
+        df = pd.DataFrame({'data': [b'some_bytes', 'some_unicode']})
         table = pa.Table.from_pandas(df)
         js = table.schema.pandas_metadata
         assert 'mixed' not in js
@@ -566,7 +559,7 @@ class TestConvertMetadata(object):
         assert table.schema.pandas_metadata['pandas_version'] is not None
 
 
-class TestConvertPrimitiveTypes(object):
+class TestConvertPrimitiveTypes:
     """
     Conversion tests for primitive (e.g. numeric) types.
     """
@@ -904,7 +897,7 @@ def test_table_integer_object_nulls_option(dtype):
     tm.assert_frame_equal(result, expected)
 
 
-class TestConvertDateTimeLikeTypes(object):
+class TestConvertDateTimeLikeTypes:
     """
     Conversion tests for datetime- and timestamp-like types (date64, etc.).
     """
@@ -993,8 +986,6 @@ class TestConvertDateTimeLikeTypes(object):
         df = pd.DataFrame({'datetime': values})
         _check_pandas_roundtrip(df)
 
-    @pytest.mark.skipif(six.PY2, reason='datetime.timezone is available since '
-                                        'python version 3.2')
     def test_python_datetime_with_timezone_tzinfo(self):
         from datetime import timezone
 
@@ -1411,11 +1402,11 @@ class TestConvertDateTimeLikeTypes(object):
 # Conversion tests for string and binary types.
 
 
-class TestConvertStringLikeTypes(object):
+class TestConvertStringLikeTypes:
 
     def test_pandas_unicode(self):
         repeats = 1000
-        values = [u'foo', None, u'bar', u'mañana', np.nan]
+        values = ['foo', None, 'bar', 'mañana', np.nan]
         df = pd.DataFrame({'strings': values * repeats})
         field = pa.field('strings', pa.string())
         schema = pa.schema([field])
@@ -1423,7 +1414,7 @@ class TestConvertStringLikeTypes(object):
         _check_pandas_roundtrip(df, expected_schema=schema)
 
     def test_bytes_to_binary(self):
-        values = [u'qux', b'foo', None, bytearray(b'barz'), 'qux', np.nan]
+        values = ['qux', b'foo', None, bytearray(b'barz'), 'qux', np.nan]
         df = pd.DataFrame({'strings': values})
 
         table = pa.Table.from_pandas(df)
@@ -1608,7 +1599,7 @@ class TestConvertStringLikeTypes(object):
             pa.array(arr, type=pa.binary(3))
 
 
-class TestConvertDecimalTypes(object):
+class TestConvertDecimalTypes:
     """
     Conversion test for decimal types.
     """
@@ -1699,7 +1690,7 @@ class TestConvertDecimalTypes(object):
         _check_pandas_roundtrip(df)
 
 
-class TestConvertListTypes(object):
+class TestConvertListTypes:
     """
     Conversion tests for list<> types.
     """
@@ -1884,7 +1875,7 @@ class TestConvertListTypes(object):
 
     def test_nested_list_first_empty(self):
         # ARROW-2711
-        data = pd.Series([[], [u"a"]])
+        data = pd.Series([[], ["a"]])
         arr = pa.array(data)
         expected = pa.array(list(data))
         assert arr.equals(expected)
@@ -1909,8 +1900,8 @@ class TestConvertListTypes(object):
         data = OrderedDict([
             ('nan_ints', [[None, 1], [2, 3]]),
             ('ints', [[0, 1], [2, 3]]),
-            ('strs', [[None, u'b'], [u'c', u'd']]),
-            ('nested_strs', [[[None, u'b'], [u'c', u'd']], None])
+            ('strs', [[None, 'b'], ['c', 'd']]),
+            ('nested_strs', [[[None, 'b'], ['c', 'd']], None])
         ])
         df = pd.DataFrame(data)
 
@@ -1945,8 +1936,8 @@ class TestConvertListTypes(object):
         ),
         (
             pa.string,
-            [[u'aaa', u'bb'], [u'c'], None],
-            [None, [u'c'], None]
+            [['aaa', 'bb'], ['c'], None],
+            [None, ['c'], None]
         ),
         (
             pa.null,
@@ -1985,7 +1976,7 @@ class TestConvertListTypes(object):
             assert result.equals(expected)
 
 
-class TestConvertStructTypes(object):
+class TestConvertStructTypes:
     """
     Conversion tests for struct types.
     """
@@ -2005,16 +1996,16 @@ class TestConvertStructTypes(object):
 
     def test_to_pandas(self):
         ints = pa.array([None, 2, 3], type=pa.int64())
-        strs = pa.array([u'a', None, u'c'], type=pa.string())
+        strs = pa.array(['a', None, 'c'], type=pa.string())
         bools = pa.array([True, False, None], type=pa.bool_())
         arr = pa.StructArray.from_arrays(
             [ints, strs, bools],
             ['ints', 'strs', 'bools'])
 
         expected = pd.Series([
-            {'ints': None, 'strs': u'a', 'bools': True},
+            {'ints': None, 'strs': 'a', 'bools': True},
             {'ints': 2, 'strs': None, 'bools': False},
-            {'ints': 3, 'strs': u'c', 'bools': None},
+            {'ints': 3, 'strs': 'c', 'bools': None},
         ])
 
         series = pd.Series(arr.to_pandas())
@@ -2097,8 +2088,7 @@ class TestConvertStructTypes(object):
 
         def iter_chunked_array(arr):
             for chunk in arr.iterchunks():
-                for item in chunk:
-                    yield item
+                yield from chunk
 
         def check(arr, data, mask=None):
             assert len(arr) == len(data)
@@ -2164,7 +2154,7 @@ class TestConvertStructTypes(object):
             expected_schema=expected_schema)
 
 
-class TestZeroCopyConversion(object):
+class TestZeroCopyConversion:
     """
     Tests that zero-copy conversion works with some types.
     """
@@ -2223,7 +2213,6 @@ class TestZeroCopyConversion(object):
         self.check_zero_copy_failure(pa.array(arr))
 
 
-# This function must be at the top-level for Python 2.7's multiprocessing
 def _non_threaded_conversion():
     df = _alltypes_example()
     _check_pandas_roundtrip(df, use_threads=False)
@@ -2236,7 +2225,7 @@ def _threaded_conversion():
     _check_pandas_roundtrip(df, use_threads=True, as_batch=True)
 
 
-class TestConvertMisc(object):
+class TestConvertMisc:
     """
     Miscellaneous conversion tests.
     """
@@ -3316,19 +3305,19 @@ def test_variable_dictionary_to_pandas():
 
 def test_dictionary_encoded_nested_to_pandas():
     # ARROW-6899
-    child = pa.array([u'a', u'a', u'a', u'b', u'b']).dictionary_encode()
+    child = pa.array(['a', 'a', 'a', 'b', 'b']).dictionary_encode()
 
     arr = pa.ListArray.from_arrays([0, 3, 5], child)
 
     result = arr.to_pandas()
-    expected = pd.Series([np.array([u'a', u'a', u'a'], dtype=object),
-                          np.array([u'b', u'b'], dtype=object)])
+    expected = pd.Series([np.array(['a', 'a', 'a'], dtype=object),
+                          np.array(['b', 'b'], dtype=object)])
 
     tm.assert_series_equal(result, expected)
 
 
 def test_dictionary_from_pandas():
-    cat = pd.Categorical([u'a', u'b', u'a'])
+    cat = pd.Categorical(['a', 'b', 'a'])
     expected_type = pa.dictionary(pa.int8(), pa.string())
 
     result = pa.array(cat)
@@ -3336,7 +3325,7 @@ def test_dictionary_from_pandas():
     assert result.type.equals(expected_type)
 
     # with missing values in categorical
-    cat = pd.Categorical([u'a', u'b', None, u'a'])
+    cat = pd.Categorical(['a', 'b', None, 'a'])
 
     result = pa.array(cat)
     assert result.to_pylist() == ['a', 'b', None, 'a']
@@ -3621,8 +3610,8 @@ def test_to_pandas_extension_dtypes_mapping():
 def test_metadata_compat_range_index_pre_0_12():
     # Forward compatibility for metadata created from pandas.RangeIndex
     # prior to pyarrow 0.13.0
-    a_values = [u'foo', u'bar', None, u'baz']
-    b_values = [u'a', u'a', u'b', u'b']
+    a_values = ['foo', 'bar', None, 'baz']
+    b_values = ['a', 'a', 'b', 'b']
     a_arrow = pa.array(a_values, type='utf8')
     b_arrow = pa.array(b_values, type='utf8')
 
@@ -3794,7 +3783,7 @@ def test_metadata_compat_missing_field_name():
     # only added later)
 
     a_values = [1, 2, 3, 4]
-    b_values = [u'a', u'b', u'c', u'd']
+    b_values = ['a', 'b', 'c', 'd']
     a_arrow = pa.array(a_values, type='int64')
     b_arrow = pa.array(b_values, type='utf8')
 

python/pyarrow/tests/test_parquet.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -22,14 +21,13 @@ import decimal
 import io
 import json
 import os
-import six
 import pickle
 import pytest
 
 import numpy as np
 
 import pyarrow as pa
-from pyarrow.compat import guid, u, BytesIO, unichar, PY2
+from pyarrow.compat import guid
 from pyarrow.pandas_compat import _pandas_api
 from pyarrow.tests import util
 from pyarrow.filesystem import LocalFileSystem, FileSystem
@@ -301,7 +299,7 @@ def test_pandas_parquet_datetime_tz():
                        'tz_eastern': s.dt.tz_convert('US/Eastern')},
                       index=s)
 
-    f = BytesIO()
+    f = io.BytesIO()
 
     arrow_table = pa.Table.from_pandas(df)
 
@@ -315,8 +313,6 @@ def test_pandas_parquet_datetime_tz():
 
 
 @pytest.mark.pandas
-@pytest.mark.skipif(six.PY2, reason='datetime.timezone is available since '
-                                    'python version 3.2')
 def test_datetime_timezone_tzinfo():
     value = datetime.datetime(2018, 1, 1, 1, 23, 45,
                               tzinfo=datetime.timezone.utc)
@@ -800,8 +796,8 @@ def test_parquet_metadata_lifetime(tempdir):
             'DOUBLE', -1.1, 4.4, 1, 4, 0
         ),
         (
-            [u'', u'b', unichar(1000), None, u'aaa'], pa.binary(),
-            'BYTE_ARRAY', b'', unichar(1000).encode('utf-8'), 1, 4, 0
+            ['', 'b', chr(1000), None, 'aaa'], pa.binary(),
+            'BYTE_ARRAY', b'', chr(1000).encode('utf-8'), 1, 4, 0
         ),
         (
             [True, False, False, True, True], pa.bool_(),
@@ -863,7 +859,7 @@ def test_statistics_convert_logical_types(tempdir):
     # (min, max, type)
     cases = [(10, 11164359321221007157, pa.uint64()),
              (10, 4294967295, pa.uint32()),
-             (u"ähnlich", u"öffentlich", pa.utf8()),
+             ("ähnlich", "öffentlich", pa.utf8()),
              (datetime.time(10, 30, 0, 1000), datetime.time(15, 30, 0, 1000),
               pa.time32('ms')),
              (datetime.time(10, 30, 0, 1000), datetime.time(15, 30, 0, 1000),
@@ -1056,15 +1052,6 @@ def test_column_of_lists(tempdir):
     table_read = _read_table(filename)
     df_read = table_read.to_pandas()
 
-    if PY2:
-        # assert_frame_equal fails when comparing datetime.date and
-        # np.datetime64, even with check_datetimelike_compat=True so
-        # convert the values to np.datetime64 instead
-        for col in ['date32[day]_list', 'date64[ms]_list']:
-            df[col] = df[col].apply(
-                lambda x: list(map(np.datetime64, x)) if x else x
-            )
-
     tm.assert_frame_equal(df, df_read)
 
 
@@ -1513,7 +1500,7 @@ def test_parquet_piece_basics():
 
 
 def test_partition_set_dictionary_type():
-    set1 = pq.PartitionSet('key1', [u('foo'), u('bar'), u('baz')])
+    set1 = pq.PartitionSet('key1', ['foo', 'bar', 'baz'])
     set2 = pq.PartitionSet('key2', [2007, 2008, 2009])
 
     assert isinstance(set1.dictionary, pa.StringArray)
@@ -1619,7 +1606,7 @@ def test_equivalency(tempdir):
         filters = [[('string', '==', b'1\0a')]]
         pq.ParquetDataset(base_path, filesystem=fs, filters=filters)
     with pytest.raises(NotImplementedError):
-        filters = [[('string', '==', u'1\0a')]]
+        filters = [[('string', '==', '1\0a')]]
         pq.ParquetDataset(base_path, filesystem=fs, filters=filters)
 
 
@@ -1879,7 +1866,7 @@ def s3_example(minio_server, s3_bucket):
     )
 
     test_dir = guid()
-    bucket_uri = 's3://{0}/{1}'.format(s3_bucket, test_dir)
+    bucket_uri = 's3://{}/{}'.format(s3_bucket, test_dir)
 
     fs.mkdir(bucket_uri)
     yield fs, bucket_uri
@@ -1948,7 +1935,7 @@ def _generate_partition_directories(fs, base_dir, partition_spec, df):
 
             level_dir = fs._path_join(
                 str(base_dir),
-                '{0}={1}'.format(name, value)
+                '{}={}'.format(name, value)
             )
             fs.mkdir(level_dir)
 
@@ -2292,7 +2279,7 @@ def test_ignore_private_directories(tempdir):
     (dirpath / '_impala_staging').mkdir()
 
     dataset = pq.ParquetDataset(dirpath)
-    assert set(map(str, paths)) == set(x.path for x in dataset.pieces)
+    assert set(map(str, paths)) == {x.path for x in dataset.pieces}
 
 
 @pytest.mark.pandas
@@ -2310,7 +2297,7 @@ def test_ignore_hidden_files_dot(tempdir):
         f.write(b'gibberish')
 
     dataset = pq.ParquetDataset(dirpath)
-    assert set(map(str, paths)) == set(x.path for x in dataset.pieces)
+    assert set(map(str, paths)) == {x.path for x in dataset.pieces}
 
 
 @pytest.mark.pandas
@@ -2328,7 +2315,7 @@ def test_ignore_hidden_files_underscore(tempdir):
         f.write(b'abcd')
 
     dataset = pq.ParquetDataset(dirpath)
-    assert set(map(str, paths)) == set(x.path for x in dataset.pieces)
+    assert set(map(str, paths)) == {x.path for x in dataset.pieces}
 
 
 @pytest.mark.pandas
@@ -2539,7 +2526,7 @@ def test_write_to_dataset_with_partitions_and_custom_filenames(tempdir):
     path = str(tempdir)
 
     def partition_filename_callback(keys):
-        return "{0}-{1}.parquet".format(*keys)
+        return "{}-{}.parquet".format(*keys)
 
     pq.write_to_dataset(output_table, path,
                         partition_by, partition_filename_callback)

python/pyarrow/tests/test_plasma.py

@@ -15,9 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 
 import multiprocessing
 import os
@@ -107,7 +104,7 @@ def assert_get_object_equal(unit_test, client1, client2, object_id,
 
 
 @pytest.mark.plasma
-class TestPlasmaClient(object):
+class TestPlasmaClient:
 
     def setup_method(self, test_method):
         import pyarrow.plasma as plasma
@@ -131,10 +128,7 @@ class TestPlasmaClient(object):
             if USE_VALGRIND:
                 time.sleep(1.0)
             self.p.send_signal(signal.SIGTERM)
-            if sys.version_info >= (3, 3):
-                self.p.wait(timeout=5)
-            else:
-                self.p.wait()
+            self.p.wait(timeout=5)
             assert self.p.returncode == 0
         finally:
             self.plasma_store_ctx.__exit__(None, None, None)
@@ -367,7 +361,7 @@ class TestPlasmaClient(object):
 
     def test_put_and_get_serialization_context(self):
 
-        class CustomType(object):
+        class CustomType:
             def __init__(self, val):
                 self.val = val
 
@@ -925,7 +919,7 @@ class TestPlasmaClient(object):
 
 
 @pytest.mark.plasma
-class TestEvictionToExternalStore(object):
+class TestEvictionToExternalStore:
 
     def setup_method(self, test_method):
         import pyarrow.plasma as plasma
@@ -943,10 +937,7 @@ class TestEvictionToExternalStore(object):
             # Check that the Plasma store is still alive.
             assert self.p.poll() is None
             self.p.send_signal(signal.SIGTERM)
-            if sys.version_info >= (3, 3):
-                self.p.wait(timeout=5)
-            else:
-                self.p.wait()
+            self.p.wait(timeout=5)
         finally:
             self.plasma_store_ctx.__exit__(None, None, None)
 

python/pyarrow/tests/test_scalars.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -21,7 +20,7 @@ import pytest
 
 import numpy as np
 
-from pyarrow.compat import unittest, u, unicode_type
+from pyarrow.compat import unittest
 import pyarrow as pa
 
 
@@ -91,14 +90,14 @@ class TestScalars(unittest.TestCase):
         assert arr[1] is pa.NA
 
     def test_string_unicode(self):
-        arr = pa.array([u'foo', None, u'mañana'])
+        arr = pa.array(['foo', None, 'mañana'])
 
         v = arr[0]
         assert isinstance(v, pa.StringValue)
-        assert v.as_py() == u'foo'
-        assert repr(v) == repr(u"foo")
-        assert str(v) == str(u"foo")
-        assert v == u'foo'
+        assert v.as_py() == 'foo'
+        assert repr(v) == repr("foo")
+        assert str(v) == "foo"
+        assert v == 'foo'
         # Assert that newly created values are equal to the previously created
         # one.
         assert v == arr[0]
@@ -106,18 +105,18 @@ class TestScalars(unittest.TestCase):
         assert arr[1] is pa.NA
 
         v = arr[2].as_py()
-        assert v == u'mañana'
-        assert isinstance(v, unicode_type)
+        assert v == 'mañana'
+        assert isinstance(v, str)
 
     def test_large_string_unicode(self):
-        arr = pa.array([u'foo', None, u'mañana'], type=pa.large_string())
+        arr = pa.array(['foo', None, 'mañana'], type=pa.large_string())
 
         v = arr[0]
         assert isinstance(v, pa.LargeStringValue)
-        assert v.as_py() == u'foo'
-        assert repr(v) == repr(u"foo")
-        assert str(v) == str(u"foo")
-        assert v == u'foo'
+        assert v.as_py() == 'foo'
+        assert repr(v) == repr("foo")
+        assert str(v) == "foo"
+        assert v == 'foo'
         # Assert that newly created values are equal to the previously created
         # one.
         assert v == arr[0]
@@ -125,11 +124,11 @@ class TestScalars(unittest.TestCase):
         assert arr[1] is pa.NA
 
         v = arr[2].as_py()
-        assert v == u'mañana'
-        assert isinstance(v, unicode_type)
+        assert v == 'mañana'
+        assert isinstance(v, str)
 
     def test_bytes(self):
-        arr = pa.array([b'foo', None, u('bar')])
+        arr = pa.array([b'foo', None, 'bar'])
 
         def check_value(v, expected):
             assert isinstance(v, pa.BinaryValue)
@@ -147,7 +146,7 @@ class TestScalars(unittest.TestCase):
         check_value(arr[2], b'bar')
 
     def test_large_bytes(self):
-        arr = pa.array([b'foo', None, u('bar')], type=pa.large_binary())
+        arr = pa.array([b'foo', None, 'bar'], type=pa.large_binary())
 
         def check_value(v, expected):
             assert isinstance(v, pa.LargeBinaryValue)
@@ -224,14 +223,14 @@ class TestScalars(unittest.TestCase):
         assert len(v) == 0
 
     def test_map(self):
-        arr = pa.array([[(u'a', 1), (u'b', 2)], None, [], [(u'c', None)]],
+        arr = pa.array([[('a', 1), ('b', 2)], None, [], [('c', None)]],
                        pa.map_(pa.string(), pa.int8()))
         v = arr[0]
         assert len(v) == 2
         assert isinstance(v, pa.MapValue)
-        assert repr(v) == repr([(u'a', 1), (u'b', 2)])
-        assert v.as_py() == [(u'a', 1), (u'b', 2)]
-        assert v[1] == (u'b', 2)
+        assert repr(v) == repr([('a', 1), ('b', 2)])
+        assert v.as_py() == [('a', 1), ('b', 2)]
+        assert v[1] == ('b', 2)
         assert v[-1] == v[1]
         assert v[-2] == v[0]
         with pytest.raises(IndexError):
@@ -244,9 +243,9 @@ class TestScalars(unittest.TestCase):
         assert len(arr[2]) == 0
 
         v = arr[3]
-        assert v == [(u'c', None)]
+        assert v == [('c', None)]
         assert len(v) == 1
-        assert v[0][0] == u'c'
+        assert v[0][0] == 'c'
         assert v[0][1] is pa.NA
 
     def test_fixed_size_list(self):
@@ -283,7 +282,7 @@ class TestScalars(unittest.TestCase):
         units = ['ns', 'us', 'ms', 's']
 
         for i, unit in enumerate(units):
-            dtype = 'datetime64[{0}]'.format(unit)
+            dtype = 'datetime64[{}]'.format(unit)
             arrow_arr = pa.Array.from_pandas(arr.astype(dtype))
             expected = pd.Timestamp('2000-01-01 12:34:56')
 
@@ -293,7 +292,7 @@ class TestScalars(unittest.TestCase):
             tz = 'America/New_York'
             arrow_type = pa.timestamp(unit, tz=tz)
 
-            dtype = 'datetime64[{0}]'.format(unit)
+            dtype = 'datetime64[{}]'.format(unit)
             arrow_arr = pa.Array.from_pandas(arr.astype(dtype),
                                              type=arrow_type)
             expected = (pd.Timestamp('2000-01-01 12:34:56')
@@ -347,7 +346,7 @@ class TestScalars(unittest.TestCase):
         units = ['us', 'ms', 's']
 
         for i, unit in enumerate(units):
-            dtype = 'timedelta64[{0}]'.format(unit)
+            dtype = 'timedelta64[{}]'.format(unit)
             arrow_arr = pa.array(arr.astype(dtype))
             expected = datetime.timedelta(seconds=60*60)
             assert isinstance(arrow_arr[1].as_py(), datetime.timedelta)

python/pyarrow/tests/test_serialization.py

@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import division
 
 import pytest
 
@@ -82,12 +81,6 @@ def assert_equal(obj1, obj2):
                                                                   .format(
                                                                       obj1,
                                                                       obj2))
-        try:
-            # Workaround to make comparison of OrderedDicts work on Python 2.7
-            if obj1 == obj2:
-                return
-        except Exception:
-            pass
         if obj1.__dict__ == {}:
             print("WARNING: Empty dict in ", obj1)
         for key in obj1.__dict__.keys():
@@ -140,13 +133,13 @@ def assert_equal(obj1, obj2):
 PRIMITIVE_OBJECTS = [
     0, 0.0, 0.9, 1 << 62, 1 << 999,
     [1 << 100, [1 << 100]], "a", string.printable, "\u262F",
-    "hello world", u"hello world", u"\xff\xfe\x9c\x001\x000\x00",
+    "hello world", "hello world", "\xff\xfe\x9c\x001\x000\x00",
     None, True, False, [], (), {}, {(1, 2): 1}, {(): 2},
-    [1, "hello", 3.0], u"\u262F", 42.0, (1.0, "hi"),
+    [1, "hello", 3.0], "\u262F", 42.0, (1.0, "hi"),
     [1, 2, 3, None], [(None,), 3, 1.0], ["h", "e", "l", "l", "o", None],
     (None, None), ("hello", None), (True, False),
     {True: "hello", False: "world"}, {"hello": "world", 1: 42, 2.5: 45},
-    {"hello": set([2, 3]), "world": set([42.0]), "this": None},
+    {"hello": {2, 3}, "world": {42.0}, "this": None},
     np.int8(3), np.int32(4), np.int64(5),
     np.uint8(3), np.uint32(4), np.uint64(5),
     np.float16(1.9), np.float32(1.9),
@@ -162,12 +155,7 @@ tensor_types = ('i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8',
                 'f2', 'f4', 'f8')
 
 
-if sys.version_info >= (3, 0):
-    PRIMITIVE_OBJECTS += [0, np.array([["hi", u"hi"], [1.3, 1]])]
-else:
-    PRIMITIVE_OBJECTS += [long(42), long(1 << 62), long(0),  # noqa
-                          np.array([["hi", u"hi"],
-                          [1.3, long(1)]])]  # noqa
+PRIMITIVE_OBJECTS += [0, np.array([["hi", "hi"], [1.3, 1]])]
 
 
 COMPLEX_OBJECTS = [
@@ -180,7 +168,7 @@ COMPLEX_OBJECTS = [
 ]
 
 
-class Foo(object):
+class Foo:
     def __init__(self, value=0):
         self.value = value
 
@@ -191,13 +179,13 @@ class Foo(object):
         return other.value == self.value
 
 
-class Bar(object):
+class Bar:
     def __init__(self):
         for i, val in enumerate(COMPLEX_OBJECTS):
             setattr(self, "field{}".format(i), val)
 
 
-class Baz(object):
+class Baz:
     def __init__(self):
         self.foo = Foo()
         self.bar = Bar()
@@ -206,7 +194,7 @@ class Baz(object):
         pass
 
 
-class Qux(object):
+class Qux:
     def __init__(self):
         self.objs = [Foo(1), Foo(42)]
 
@@ -300,7 +288,7 @@ def large_memory_map(tmpdir_factory, size=100*1024*1024):
 def test_clone():
     context = pa.SerializationContext()
 
-    class Foo(object):
+    class Foo:
         pass
 
     def custom_serializer(obj):
@@ -497,7 +485,7 @@ def test_numpy_base_object(tmpdir):
 # see https://issues.apache.org/jira/browse/ARROW-1695
 def test_serialization_callback_numpy():
 
-    class DummyClass(object):
+    class DummyClass:
         pass
 
     def serialize_dummy_class(obj):
@@ -677,7 +665,7 @@ def test_scipy_sparse_csr_matrix_serialization():
 @pytest.mark.filterwarnings(
     "ignore:the matrix subclass:PendingDeprecationWarning")
 def test_numpy_matrix_serialization(tmpdir):
-    class CustomType(object):
+    class CustomType:
         def __init__(self, val):
             self.val = val
 
@@ -705,17 +693,17 @@ def test_pyarrow_objects_serialization(large_buffer):
     # NOTE: We have to put these objects inside,
     # or it will affect 'test_total_bytes_allocated'.
     pyarrow_objects = [
-        pa.array([1, 2, 3, 4]), pa.array(['1', u'never U+1F631', '',
-                                         u"233 * U+1F600"]),
+        pa.array([1, 2, 3, 4]), pa.array(['1', 'never U+1F631', '',
+                                         "233 * U+1F600"]),
         pa.array([1, None, 2, 3]),
         pa.Tensor.from_numpy(np.random.rand(2, 3, 4)),
         pa.RecordBatch.from_arrays(
             [pa.array([1, None, 2, 3]),
-             pa.array(['1', u'never U+1F631', '', u"233 * u1F600"])],
+             pa.array(['1', 'never U+1F631', '', "233 * u1F600"])],
             ['a', 'b']),
         pa.Table.from_arrays([pa.array([1, None, 2, 3]),
-                              pa.array(['1', u'never U+1F631', '',
-                                       u"233 * u1F600"])],
+                              pa.array(['1', 'never U+1F631', '',
+                                       "233 * u1F600"])],
                              ['a', 'b'])
     ]
     for obj in pyarrow_objects:
@@ -724,7 +712,7 @@ def test_pyarrow_objects_serialization(large_buffer):
 
 def test_buffer_serialization():
 
-    class BufferClass(object):
+    class BufferClass:
         pass
 
     def serialize_buffer_class(obj):
@@ -774,7 +762,7 @@ def test_arrow_limits(self):
 
 def test_serialization_callback_error():
 
-    class TempClass(object):
+    class TempClass:
         pass
 
     # Pass a SerializationContext into serialize, but TempClass
@@ -795,7 +783,7 @@ def test_serialization_callback_error():
         serialized_object.deserialize(deserialization_context)
     assert err.value.type_id == "TempClass"
 
-    class TempClass2(object):
+    class TempClass2:
         pass
 
     # Make sure that we receive an error when we use an inappropriate value for
@@ -825,7 +813,7 @@ def test_fallback_to_subclasses():
     assert type(reconstructed_object) == Foo
 
 
-class Serializable(object):
+class Serializable:
     pass
 
 
@@ -907,7 +895,7 @@ def test_deserialize_components_in_different_process():
 
         import pyarrow as pa
 
-        data = {0!r}
+        data = {!r}
         components = pickle.loads(data)
         arr = pa.deserialize_components(components)
 
@@ -974,7 +962,7 @@ def test_deserialize_buffer_in_different_process():
 
 def test_set_pickle():
     # Use a custom type to trigger pickling.
-    class Foo(object):
+    class Foo:
         pass
 
     context = pa.SerializationContext()
@@ -1053,7 +1041,7 @@ def test_serialization_determinism():
 
 
 def test_serialize_recursive_objects():
-    class ClassA(object):
+    class ClassA:
         pass
 
     # Make a list that contains itself.

python/pyarrow/tests/test_table.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -1002,9 +1001,9 @@ def test_invalid_table_construct():
 
 
 @pytest.mark.parametrize('data, klass', [
-    (([u'', u'foo', u'bar'], [4.5, 5, None]), list),
-    (([u'', u'foo', u'bar'], [4.5, 5, None]), pa.array),
-    (([[u''], [u'foo', u'bar']], [[4.5], [5., None]]), pa.chunked_array),
+    ((['', 'foo', 'bar'], [4.5, 5, None]), list),
+    ((['', 'foo', 'bar'], [4.5, 5, None]), pa.array),
+    (([[''], ['foo', 'bar']], [[4.5], [5., None]]), pa.chunked_array),
 ])
 def test_from_arrays_schema(data, klass):
     data = [klass(data[0]), klass(data[1])]
@@ -1051,7 +1050,7 @@ def test_table_from_pydict():
     schema = pa.schema([('strs', pa.utf8()), ('floats', pa.float64())])
 
     # With lists as values
-    data = OrderedDict([('strs', [u'', u'foo', u'bar']),
+    data = OrderedDict([('strs', ['', 'foo', 'bar']),
                         ('floats', [4.5, 5, None])])
     table = pa.Table.from_pydict(data)
     assert table.num_columns == 2
@@ -1099,8 +1098,8 @@ def test_table_from_pydict():
 
 
 @pytest.mark.parametrize('data, klass', [
-    (([u'', u'foo', u'bar'], [4.5, 5, None]), pa.array),
-    (([[u''], [u'foo', u'bar']], [[4.5], [5., None]]), pa.chunked_array),
+    ((['', 'foo', 'bar'], [4.5, 5, None]), pa.array),
+    (([[''], ['foo', 'bar']], [[4.5], [5., None]]), pa.chunked_array),
 ])
 def test_table_from_pydict_arrow_arrays(data, klass):
     data = OrderedDict([('strs', klass(data[0])), ('floats', klass(data[1]))])
@@ -1133,9 +1132,9 @@ def test_table_from_pydict_arrow_arrays(data, klass):
 
 
 @pytest.mark.parametrize('data, klass', [
-    (([u'', u'foo', u'bar'], [4.5, 5, None]), list),
-    (([u'', u'foo', u'bar'], [4.5, 5, None]), pa.array),
-    (([[u''], [u'foo', u'bar']], [[4.5], [5., None]]), pa.chunked_array),
+    ((['', 'foo', 'bar'], [4.5, 5, None]), list),
+    ((['', 'foo', 'bar'], [4.5, 5, None]), pa.array),
+    (([[''], ['foo', 'bar']], [[4.5], [5., None]]), pa.chunked_array),
 ])
 def test_table_from_pydict_schema(data, klass):
     # passed schema is source of truth for the columns
@@ -1161,7 +1160,7 @@ def test_table_from_pandas_schema():
     # passed schema is source of truth for the columns
     import pandas as pd
 
-    df = pd.DataFrame(OrderedDict([('strs', [u'', u'foo', u'bar']),
+    df = pd.DataFrame(OrderedDict([('strs', ['', 'foo', 'bar']),
                                    ('floats', [4.5, 5, None])]))
 
     # with different but compatible schema

python/pyarrow/tests/test_tensor.py

@@ -184,8 +184,6 @@ def test_read_tensor(tmpdir):
     np.testing.assert_equal(data, array)
 
 
-@pytest.mark.skipif(sys.version_info < (3,),
-                    reason="requires Python 3+")
 def test_tensor_memoryview():
     # Tensors support the PEP 3118 buffer protocol
     for dtype, expected_format in [(np.int8, '=b'),

python/pyarrow/types.pxi

@@ -176,13 +176,7 @@ cdef class DataType:
         """
         cdef DataType other_type
 
-        if not isinstance(other, DataType):
-            if not isinstance(other, six.string_types):
-                raise TypeError(other)
-            other_type = type_for_alias(other)
-        else:
-            other_type = other
-
+        other_type = ensure_type(other)
         return self.type.Equals(deref(other_type.type))
 
     def to_pandas_dtype(self):
@@ -384,9 +378,9 @@ cdef class StructType(DataType):
         """
         Return the struct field with the given index or name.
         """
-        if isinstance(i, six.string_types):
+        if isinstance(i, (bytes, str)):
             return self.field_by_name(i)
-        elif isinstance(i, six.integer_types):
+        elif isinstance(i, int):
             return self.child(i)
         else:
             raise TypeError('Expected integer or string index')
@@ -1096,13 +1090,13 @@ cdef class Schema:
         -------
         pyarrow.Field
         """
-        if isinstance(i, six.string_types):
+        if isinstance(i, (bytes, str)):
             field_index = self.get_field_index(i)
             if field_index < 0:
                 raise KeyError("Column {} does not exist in schema".format(i))
             else:
                 return self._field(field_index)
-        elif isinstance(i, six.integer_types):
+        elif isinstance(i, int):
             return self._field(i)
         else:
             raise TypeError("Index must either be string or integer")
@@ -1524,7 +1518,7 @@ def tzinfo_to_string(tz):
     elif isinstance(tz, pytz._FixedOffset):
         return fixed_offset_to_string(tz)
     elif isinstance(tz, datetime.tzinfo):
-        if six.PY3 and isinstance(tz, datetime.timezone):
+        if isinstance(tz, datetime.timezone):
             return fixed_offset_to_string(tz)
         else:
             raise ValueError('Unable to convert timezone `{}` to string'
@@ -1607,7 +1601,7 @@ def timestamp(unit, tz=None):
             return _timestamp_type_cache[unit_code]
         _timestamp_type_cache[unit_code] = out
     else:
-        if not isinstance(tz, six.string_types):
+        if not isinstance(tz, (bytes, str)):
             tz = tzinfo_to_string(tz)
 
         c_timezone = tobytes(tz)
@@ -2157,7 +2151,7 @@ cdef DataType ensure_type(object ty, c_bool allow_none=False):
         return None
     elif isinstance(ty, DataType):
         return ty
-    elif isinstance(ty, six.string_types):
+    elif isinstance(ty, str):
         return type_for_alias(ty)
     else:
         raise TypeError('DataType expected, got {!r}'.format(type(ty)))

python/pyarrow/types.py

@@ -17,7 +17,6 @@
 
 # Tools for dealing with Arrow type metadata in Python
 
-from __future__ import absolute_import
 
 from pyarrow.lib import (is_boolean_value,  # noqa
                          is_integer_value,

python/pyarrow/util.py

@@ -17,32 +17,13 @@
 
 # Miscellaneous utility code
 
-from __future__ import absolute_import
-
 import contextlib
 import functools
-import six
+import pathlib
 import socket
 import warnings
 
 
-try:
-    from textwrap import indent
-except ImportError:
-    def indent(text, prefix):
-        return ''.join(prefix + line for line in text.splitlines(True))
-
-try:
-    # pathlib might not be available
-    try:
-        import pathlib
-    except ImportError:
-        import pathlib2 as pathlib  # python 2 backport
-    _has_pathlib = True
-except ImportError:
-    _has_pathlib = False
-
-
 def implements(f):
     def decorator(g):
         g.__doc__ = f.__doc__
@@ -51,7 +32,7 @@ def implements(f):
 
 
 def _deprecate_api(old_name, new_name, api, next_version):
-    msg = ('pyarrow.{0} is deprecated as of {1}, please use {2} instead'
+    msg = ('pyarrow.{} is deprecated as of {}, please use {} instead'
            .format(old_name, next_version, new_name))
 
     def wrapper(*args, **kwargs):
@@ -63,16 +44,16 @@ def _deprecate_api(old_name, new_name, api, next_version):
 def _is_path_like(path):
     # PEP519 filesystem path protocol is available from python 3.6, so pathlib
     # doesn't implement __fspath__ for earlier versions
-    return (isinstance(path, six.string_types) or
+    return (isinstance(path, str) or
             hasattr(path, '__fspath__') or
-            (_has_pathlib and isinstance(path, pathlib.Path)))
+            isinstance(path, pathlib.Path))
 
 
 def _stringify_path(path):
     """
     Convert *path* to a string or unicode path if possible.
     """
-    if isinstance(path, six.string_types):
+    if isinstance(path, str):
         return path
 
     # checking whether path implements the filesystem protocol
@@ -80,7 +61,7 @@ def _stringify_path(path):
         return path.__fspath__()  # new in python 3.6
     except AttributeError:
         # fallback pathlib ckeck for earlier python versions than 3.6
-        if _has_pathlib and isinstance(path, pathlib.Path):
+        if isinstance(path, pathlib.Path):
             return str(path)
 
     raise TypeError("not a path-like object")

python/requirements-test.txt

@@ -1,7 +1,6 @@
 cython
 hypothesis
 pandas
-pathlib2; python_version < "3.4"
 pickle5; python_version == "3.6" or python_version == "3.7"
 pytest
 pytest-lazy-fixture

python/requirements-wheel.txt

@@ -1,10 +1,8 @@
 Keras-Preprocessing
 cython
-futures; python_version < "3.2"
 numpy>=1.16
 pandas
 setuptools_scm==3.2.0
-six>=1.0.0
 # TODO: TensorFlow doesn't support Python 3.8 yet.
 # See https://github.com/tensorflow/tensorflow/issues/33374
 tensorflow; python_version >= "3.0" and python_version < "3.8"

python/requirements.txt

@@ -1,4 +1,2 @@
 six>=1.0.0
 numpy>=1.14
-futures; python_version < "3.2"
-enum34 >= 1.1.6; python_version < "3.4"

python/setup.py

@@ -559,8 +559,6 @@ class BinaryDistribution(Distribution):
 install_requires = (
     'numpy >= 1.14',
     'six >= 1.0.0',
-    'futures; python_version < "3.2"',
-    'enum34 >= 1.1.6; python_version < "3.4"',
 )
 
 
@@ -597,8 +595,7 @@ setup(
     },
     setup_requires=['setuptools_scm', 'cython >= 0.29'] + setup_requires,
     install_requires=install_requires,
-    tests_require=['pytest', 'pandas', 'hypothesis',
-                   'pathlib2; python_version < "3.4"'],
+    tests_require=['pytest', 'pandas', 'hypothesis'],
     description='Python library for Apache Arrow',
     long_description=long_description,
     long_description_content_type='text/markdown',