Get Python test suite passing again

Change-Id: I9066fe7688f18aed9b72ce84004f5df957585ecb

cpp/src/arrow/python/CMakeLists.txt

@@ -50,12 +50,12 @@ set(ARROW_PYTHON_TEST_LINK_LIBS ${ARROW_PYTHON_MIN_TEST_LIBS})
 # ----------------------------------------------------------------------
 
 set(ARROW_PYTHON_SRCS
+  builtin_convert.cc
   common.cc
   config.cc
   helpers.cc
   io.cc
-  adapters/builtin.cc
-  adapters/pandas.cc
+  pandas_convert.cc
 )
 
 set(ARROW_PYTHON_SHARED_LINK_LIBS
@@ -73,21 +73,21 @@ ADD_ARROW_LIB(arrow_python
 
 install(FILES
   api.h
+  builtin_convert.h
   common.h
   config.h
   do_import_numpy.h
-  helper.h
+  helpers.h
   io.h
   numpy_interop.h
+  pandas_convert.h
   type_traits.h
-  adapters/builtin.h
-  adapters/pandas.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/python")
 
 # set_target_properties(arrow_python_shared PROPERTIES
 #     INSTALL_RPATH "\$ORIGIN")
 
 if (ARROW_BUILD_TESTS)
-  ADD_ARROW_TEST(adapters/pandas-test
+  ADD_ARROW_TEST(pandas-test
     STATIC_LINK_LIBS "${ARROW_PYTHON_TEST_LINK_LIBS}")
 endif()

cpp/src/arrow/python/api.h

@@ -18,9 +18,10 @@
 #ifndef ARROW_PYTHON_API_H
 #define ARROW_PYTHON_API_H
 
+#include "arrow/python/builtin_convert.h"
+#include "arrow/python/common.h"
 #include "arrow/python/helpers.h"
-
-#include "arrow/python/adapters/builtin.h"
-#include "arrow/python/adapters/pandas.h"
+#include "arrow/python/io.h"
+#include "arrow/python/pandas_convert.h"
 
 #endif  // ARROW_PYTHON_API_H

cpp/src/arrow/python/builtin_convert.cc

@@ -19,7 +19,7 @@
 #include <datetime.h>
 #include <sstream>
 
-#include "arrow/python/adapters/builtin.h"
+#include "arrow/python/builtin_convert.h"
 
 #include "arrow/api.h"
 #include "arrow/status.h"

cpp/src/arrow/python/pandas-test.cc

@@ -24,7 +24,7 @@
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
-#include "arrow/python/adapters/pandas.h"
+#include "arrow/python/pandas_convert.h"
 #include "arrow/schema.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"

cpp/src/arrow/python/pandas_convert.cc

@@ -19,8 +19,8 @@
 
 #include <Python.h>
 
-#include "arrow/python/adapters/pandas.h"
 #include "arrow/python/numpy_interop.h"
+#include "arrow/python/pandas_convert.h"
 
 #include <algorithm>
 #include <atomic>
@@ -36,6 +36,11 @@
 #include "arrow/array.h"
 #include "arrow/column.h"
 #include "arrow/loader.h"
+#include "arrow/python/builtin_convert.h"
+#include "arrow/python/common.h"
+#include "arrow/python/config.h"
+#include "arrow/python/type_traits.h"
+#include "arrow/python/util/datetime.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type_fwd.h"
@@ -43,12 +48,6 @@
 #include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
 
-#include "arrow/python/adapters/builtin.h"
-#include "arrow/python/common.h"
-#include "arrow/python/config.h"
-#include "arrow/python/type_traits.h"
-#include "arrow/python/util/datetime.h"
-
 namespace arrow {
 namespace py {
 

python/CMakeLists.txt

@@ -72,6 +72,8 @@ endif(CCACHE_FOUND)
 # Compiler flags
 ############################################################
 
+include(BuildUtils)
+include(CompilerInfo)
 include(SetupCxxFlags)
 
 # Add common flags
@@ -83,8 +85,6 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
 # Suppress Cython warnings
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable")
 
-# Determine compiler version
-include(CompilerInfo)
 
 if ("${COMPILER_FAMILY}" STREQUAL "clang")
   # Using Clang with ccache causes a bunch of spurious warnings that are

python/cmake_modules/FindArrow.cmake

@@ -68,6 +68,7 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_IO_LIB_NAME libarrow_io)
   set(ARROW_IPC_LIB_NAME libarrow_ipc)
   set(ARROW_JEMALLOC_LIB_NAME libarrow_jemalloc)
+  set(ARROW_PYTHON_LIB_NAME libarrow_python)
 
   set(ARROW_LIBS ${ARROW_SEARCH_LIB_PATH})
   set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a)
@@ -80,8 +81,7 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_IPC_SHARED_LIB ${ARROW_LIBS}/${ARROW_IPC_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 
   set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_JEMALLOC_LIB_NAME}.a)
-  set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/${ARROW_JEMALLOC_LIB_NAME}${CMAKE_SHARED_LIBRARY
-_SUFFIX})
+  set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/${ARROW_JEMALLOC_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 
   set(ARROW_PYTHON_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_PYTHON_LIB_NAME}.a)
   set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})

python/pyarrow/config.pyx

@@ -14,21 +14,21 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-cdef extern from 'pyarrow/do_import_numpy.h':
+cdef extern from 'arrow/python/do_import_numpy.h':
     pass
 
-cdef extern from 'pyarrow/numpy_interop.h' namespace 'arrow::py':
+cdef extern from 'arrow/python/numpy_interop.h' namespace 'arrow::py':
     int import_numpy()
 
-cdef extern from 'pyarrow/config.h' namespace 'arrow::py':
-    void pyarrow_init()
-    void pyarrow_set_numpy_nan(object o)
+cdef extern from 'arrow/python/config.h' namespace 'arrow::py':
+    void Init()
+    void set_numpy_nan(object o)
 
 import_numpy()
-pyarrow_init()
+Init()
 
 import numpy as np
-pyarrow_set_numpy_nan(np.nan)
+set_numpy_nan(np.nan)
 
 import multiprocessing
 import os

python/pyarrow/includes/pyarrow.pxd

@@ -25,7 +25,7 @@ from pyarrow.includes.libarrow cimport (CArray, CBuffer, CColumn,
 cimport pyarrow.includes.libarrow_io as arrow_io
 
 
-cdef extern from "pyarrow/api.h" namespace "arrow::py" nogil:
+cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
     shared_ptr[CDataType] GetPrimitiveType(Type type)
     shared_ptr[CDataType] GetTimestampType(TimeUnit unit)
     CStatus ConvertPySequence(object obj, CMemoryPool* pool,
@@ -53,13 +53,9 @@ cdef extern from "pyarrow/api.h" namespace "arrow::py" nogil:
     void set_default_memory_pool(CMemoryPool* pool)
     CMemoryPool* get_memory_pool()
 
-
-cdef extern from "pyarrow/common.h" namespace "arrow::py" nogil:
     cdef cppclass PyBuffer(CBuffer):
         PyBuffer(object o)
 
-
-cdef extern from "pyarrow/io.h" namespace "arrow::py" nogil:
     cdef cppclass PyReadableFile(arrow_io.RandomAccessFile):
         PyReadableFile(object fo)
 

python/setup.py

@@ -186,7 +186,7 @@ class build_ext(_build_ext):
             # a bit hacky
             build_lib = saved_cwd
 
-        # Move the built libpyarrow library to the place expected by the Python
+        # Move the libraries to the place expected by the Python
         # build
         shared_library_prefix = 'lib'
         if sys.platform == 'darwin':
@@ -203,15 +203,16 @@ class build_ext(_build_ext):
             pass
 
         def move_lib(lib_name):
-            lib_filename = shared_library_prefix + lib_name + shared_library_suffix
+            lib_filename = (shared_library_prefix + lib_name +
+                            shared_library_suffix)
             shutil.move(pjoin(self.build_type, lib_filename),
                         pjoin(build_lib, 'pyarrow', lib_filename))
 
-        move_lib("pyarrow")
         if self.bundle_arrow_cpp:
             move_lib("arrow")
             move_lib("arrow_io")
             move_lib("arrow_ipc")
+            move_lib("arrow_python")
             if self.with_jemalloc:
                 move_lib("arrow_jemalloc")
             if self.with_parquet:
@@ -227,14 +228,14 @@ class build_ext(_build_ext):
                 if self._failure_permitted(name):
                     print('Cython module {0} failure permitted'.format(name))
                     continue
-                raise RuntimeError('libpyarrow C-extension failed to build:',
+                raise RuntimeError('pyarrow C-extension failed to build:',
                                    os.path.abspath(built_path))
 
             ext_path = pjoin(build_lib, self._get_cmake_ext_path(name))
             if os.path.exists(ext_path):
                 os.remove(ext_path)
             self.mkpath(os.path.dirname(ext_path))
-            print('Moving built libpyarrow C-extension', built_path,
+            print('Moving built C-extension', built_path,
                   'to build path', ext_path)
             shutil.move(self.get_ext_built(name), ext_path)
             self._found_names.append(name)