Start moving python/src/pyarrow tp cpp/src/arrow/python

Change-Id: Ia7c31d5c7cbc7de5b6ad65e53120918ddeba2db9

cpp/CMakeLists.txt

@@ -106,6 +106,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Rely on boost shared libraries where relevant"
     ON)
 
+  option(ARROW_PYTHON
+    "Build the Arrow CPython extensions"
+    OFF)
+
   option(ARROW_SSE3
     "Build Arrow with SSE3"
     ON)
@@ -908,3 +912,14 @@ if(ARROW_IPC)
 
   add_subdirectory(src/arrow/ipc)
 endif()
+
+if(ARROW_PYTHON)
+  find_package(PythonLibsNew REQUIRED)
+  find_package(NumPy REQUIRED)
+
+  include_directories(SYSTEM
+    ${NUMPY_INCLUDE_DIRS}
+    ${PYTHON_INCLUDE_DIRS})
+
+  add_subdirectory(src/arrow/python)
+endif()

cpp/src/arrow/python/CMakeLists.txt

@@ -0,0 +1,117 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+
+#######################################
+# arrow_python
+#######################################
+# Add a new test case, with or without an executable that should be built.
+#
+# REL_TEST_NAME is the name of the test. It may be a single component
+# (e.g. monotime-test) or contain additional components (e.g.
+# net/net_util-test). Either way, the last component must be a globally
+# unique name.
+#
+# Arguments after the test name will be passed to set_tests_properties().
+function(ADD_ARROW_PYTHON_TEST REL_TEST_NAME)
+  if(NO_TESTS)
+    return()
+  endif()
+  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME}.cc)
+    # This test has a corresponding .cc file, set it up as an executable.
+    set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
+    add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
+    target_link_libraries(${TEST_NAME} ${PYARROW_TEST_LINK_LIBS})
+  else()
+    # No executable, just invoke the test (probably a script) directly.
+    set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
+  endif()
+
+  add_test(${TEST_NAME}
+    ${BUILD_SUPPORT_DIR}/run-test.sh ${TEST_PATH})
+  if(ARGN)
+    set_tests_properties(${TEST_NAME} PROPERTIES ${ARGN})
+  endif()
+endfunction()
+
+# A wrapper for add_dependencies() that is compatible with NO_TESTS.
+function(ADD_ARROW_PYTHON_TEST_DEPENDENCIES REL_TEST_NAME)
+  if(NO_TESTS)
+    return()
+  endif()
+  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+  add_dependencies(${TEST_NAME} ${ARGN})
+endfunction()
+
+if (ARROW_BUILD_TESTS)
+  add_library(pyarrow_test_main STATIC
+	util/test_main.cc)
+
+  if (APPLE)
+	target_link_libraries(pyarrow_test_main
+      gtest
+      dl)
+	set_target_properties(pyarrow_test_main
+      PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+  else()
+	target_link_libraries(pyarrow_test_main
+      gtest
+      pthread
+      dl
+	  )
+  endif()
+endif()
+
+# ----------------------------------------------------------------------
+
+set(ARROW_PYTHON_SRCS
+  common.cc
+  config.cc
+  helpers.cc
+  io.cc
+  adapters/builtin.cc
+  adapters/pandas.cc
+)
+
+set(ARROW_PYTHON_SHARED_LINK_LIBS
+  arrow_io_shared
+  arrow_ipc_shared
+  arrow_shared
+)
+
+set(ARROW_PYTHON_TEST_LINK_LIBS
+  arrow_ipc_static
+  arrow_io_static)
+
+ADD_ARROW_LIB(arrow_python
+  SOURCES ${ARROW_PYTHON_SRCS}
+  SHARED_LINK_FLAGS ""
+  SHARED_LINK_LIBS ${ARROW_PYTHON_SHARED_LINK_LIBS}
+  STATIC_LINK_LIBS ${ARROW_IO_SHARED_PRIVATE_LINK_LIBS}
+)
+
+install(FILES
+  api.h
+  common.h
+  config.h
+  helper.h
+  io.h
+  numpy_interop.h
+  adapters/builtin.h
+  adapters/pandas.h
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/python")
+
+# set_target_properties(pyarrow PROPERTIES
+#     INSTALL_RPATH "\$ORIGIN")
+
+ADD_ARROW_PYTHON_TEST(adapters/pandas-test)

cpp/src/arrow/python/adapters/builtin.cc

@@ -19,13 +19,13 @@
 #include <datetime.h>
 #include <sstream>
 
-#include "pyarrow/adapters/builtin.h"
+#include "arrow/python/adapters/builtin.h"
 
 #include "arrow/api.h"
 #include "arrow/status.h"
 
-#include "pyarrow/helpers.h"
-#include "pyarrow/util/datetime.h"
+#include "arrow/python/helpers.h"
+#include "arrow/python/util/datetime.h"
 
 namespace arrow {
 namespace py {

cpp/src/arrow/python/adapters/builtin.h

@@ -18,8 +18,8 @@
 // Functions for converting between CPython built-in data structures and Arrow
 // data structures
 
-#ifndef PYARROW_ADAPTERS_BUILTIN_H
-#define PYARROW_ADAPTERS_BUILTIN_H
+#ifndef ARROW_PYTHON_ADAPTERS_BUILTIN_H
+#define ARROW_PYTHON_ADAPTERS_BUILTIN_H
 
 #include <Python.h>
 
@@ -29,7 +29,7 @@
 
 #include "arrow/util/visibility.h"
 
-#include "pyarrow/common.h"
+#include "arrow/python/common.h"
 
 namespace arrow {
 
@@ -51,4 +51,4 @@ Status ConvertPySequence(PyObject* obj, MemoryPool* pool, std::shared_ptr<Array>
 }  // namespace py
 }  // namespace arrow
 
-#endif  // PYARROW_ADAPTERS_BUILTIN_H
+#endif  // ARROW_PYTHON_ADAPTERS_BUILTIN_H

cpp/src/arrow/python/adapters/pandas-test.cc

@@ -28,7 +28,7 @@
 #include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "pyarrow/adapters/pandas.h"
+#include "arrow/python/adapters/pandas.h"
 
 namespace arrow {
 namespace py {

cpp/src/arrow/python/adapters/pandas.cc

@@ -19,8 +19,8 @@
 
 #include <Python.h>
 
-#include "pyarrow/adapters/pandas.h"
-#include "pyarrow/numpy_interop.h"
+#include "arrow/python/adapters/pandas.h"
+#include "arrow/python/numpy_interop.h"
 
 #include <algorithm>
 #include <atomic>
@@ -43,11 +43,11 @@
 #include "arrow/util/bit-util.h"
 #include "arrow/util/macros.h"
 
-#include "pyarrow/adapters/builtin.h"
-#include "pyarrow/common.h"
-#include "pyarrow/config.h"
-#include "pyarrow/type_traits.h"
-#include "pyarrow/util/datetime.h"
+#include "arrow/python/adapters/builtin.h"
+#include "arrow/python/common.h"
+#include "arrow/python/config.h"
+#include "arrow/python/type_traits.h"
+#include "arrow/python/util/datetime.h"
 
 namespace arrow {
 namespace py {

cpp/src/arrow/python/adapters/pandas.h

@@ -18,8 +18,8 @@
 // Functions for converting between pandas's NumPy-based data representation
 // and Arrow data structures
 
-#ifndef PYARROW_ADAPTERS_PANDAS_H
-#define PYARROW_ADAPTERS_PANDAS_H
+#ifndef ARROW_PYTHON_ADAPTERS_PANDAS_H
+#define ARROW_PYTHON_ADAPTERS_PANDAS_H
 
 #include <Python.h>
 
@@ -76,4 +76,4 @@ Status PandasObjectsToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
 }  // namespace py
 }  // namespace arrow
 
-#endif  // PYARROW_ADAPTERS_PANDAS_H
+#endif  // ARROW_PYTHON_ADAPTERS_PANDAS_H

cpp/src/arrow/python/api.h

@@ -15,12 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef PYARROW_API_H
-#define PYARROW_API_H
+#ifndef ARROW_PYTHON_API_H
+#define ARROW_PYTHON_API_H
 
-#include "pyarrow/helpers.h"
+#include "arrow/python/helpers.h"
 
-#include "pyarrow/adapters/builtin.h"
-#include "pyarrow/adapters/pandas.h"
+#include "arrow/python/adapters/builtin.h"
+#include "arrow/python/adapters/pandas.h"
 
-#endif  // PYARROW_API_H
+#endif  // ARROW_PYTHON_API_H

cpp/src/arrow/python/common.cc

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "pyarrow/common.h"
+#include "arrow/python/common.h"
 
 #include <cstdlib>
 #include <mutex>
@@ -57,7 +57,7 @@ PyBuffer::PyBuffer(PyObject* obj)
         capacity_ = buffer->len;
         is_mutable_ = false;
         Py_INCREF(obj_);
-    } 
+    }
 }
 
 PyBuffer::~PyBuffer() {

cpp/src/arrow/python/common.h

@@ -15,10 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef PYARROW_COMMON_H
-#define PYARROW_COMMON_H
+#ifndef ARROW_PYTHON_COMMON_H
+#define ARROW_PYTHON_COMMON_H
 
-#include "pyarrow/config.h"
+#include "arrow/python/config.h"
 
 #include "arrow/buffer.h"
 #include "arrow/util/macros.h"
@@ -134,4 +134,4 @@ class ARROW_EXPORT PyBuffer : public Buffer {
 }  // namespace py
 }  // namespace arrow
 
-#endif  // PYARROW_COMMON_H
+#endif  // ARROW_PYTHON_COMMON_H

cpp/src/arrow/python/config.h

@@ -15,15 +15,14 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef PYARROW_CONFIG_H
-#define PYARROW_CONFIG_H
+#ifndef ARROW_PYTHON_CONFIG_H
+#define ARROW_PYTHON_CONFIG_H
 
 #include <Python.h>
 
+#include "arrow/python/numpy_interop.h"
 #include "arrow/util/visibility.h"
 
-#include "pyarrow/numpy_interop.h"
-
 #if PY_MAJOR_VERSION >= 3
 #define PyString_Check PyUnicode_Check
 #endif
@@ -43,4 +42,4 @@ void pyarrow_set_numpy_nan(PyObject* obj);
 }  // namespace py
 }  // namespace arrow
 
-#endif  // PYARROW_CONFIG_H
+#endif  // ARROW_PYTHON_CONFIG_H

cpp/src/arrow/python/helpers.cc

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "pyarrow/helpers.h"
+#include "arrow/python/helpers.h"
 
 #include <arrow/api.h>
 

python/CMakeLists.txt

@@ -57,7 +57,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build the PyArrow jemalloc integration"
     OFF)
   option(PYARROW_BUNDLE_ARROW_CPP
-    "Bundle the Arrow C++ libraries" 
+    "Bundle the Arrow C++ libraries"
     OFF)
 endif()
 
@@ -215,53 +215,6 @@ include_directories(SYSTEM
   ${PYTHON_INCLUDE_DIRS}
   src)
 
-############################################################
-# Testing
-############################################################
-
-# Add a new test case, with or without an executable that should be built.
-#
-# REL_TEST_NAME is the name of the test. It may be a single component
-# (e.g. monotime-test) or contain additional components (e.g.
-# net/net_util-test). Either way, the last component must be a globally
-# unique name.
-#
-# Arguments after the test name will be passed to set_tests_properties().
-function(ADD_PYARROW_TEST REL_TEST_NAME)
-  if(NO_TESTS)
-    return()
-  endif()
-  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME}.cc)
-    # This test has a corresponding .cc file, set it up as an executable.
-    set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
-    add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
-    target_link_libraries(${TEST_NAME} ${PYARROW_TEST_LINK_LIBS})
-  else()
-    # No executable, just invoke the test (probably a script) directly.
-    set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
-  endif()
-
-  add_test(${TEST_NAME}
-    ${BUILD_SUPPORT_DIR}/run-test.sh ${TEST_PATH})
-  if(ARGN)
-    set_tests_properties(${TEST_NAME} PROPERTIES ${ARGN})
-  endif()
-endfunction()
-
-# A wrapper for add_dependencies() that is compatible with NO_TESTS.
-function(ADD_PYARROW_TEST_DEPENDENCIES REL_TEST_NAME)
-  if(NO_TESTS)
-    return()
-  endif()
-  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
-  add_dependencies(${TEST_NAME} ${ARGN})
-endfunction()
-
-enable_testing()
-
 ############################################################
 # Dependencies
 ############################################################
@@ -429,9 +382,6 @@ if (UNIX)
   set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
 endif()
 
-add_subdirectory(src/pyarrow)
-add_subdirectory(src/pyarrow/util)
-
 set(CYTHON_EXTENSIONS
   array
   config
@@ -444,15 +394,6 @@ set(CYTHON_EXTENSIONS
   table
 )
 
-set(PYARROW_SRCS
-  src/pyarrow/common.cc
-  src/pyarrow/config.cc
-  src/pyarrow/helpers.cc
-  src/pyarrow/io.cc
-  src/pyarrow/adapters/builtin.cc
-  src/pyarrow/adapters/pandas.cc
-)
-
 set(LINK_LIBS
   arrow
   arrow_io
@@ -503,18 +444,6 @@ if (PYARROW_BUILD_JEMALLOC)
     jemalloc)
 endif()
 
-add_library(pyarrow SHARED
-  ${PYARROW_SRCS})
-if (PYARROW_BUNDLE_ARROW_CPP)
-  set_target_properties(pyarrow PROPERTIES
-      INSTALL_RPATH "\$ORIGIN")
-endif()
-target_link_libraries(pyarrow ${LINK_LIBS})
-
-if(APPLE)
-  set_target_properties(pyarrow PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
-endif()
-
 ############################################################
 # Setup and build Cython modules
 ############################################################

python/cmake_modules/FindArrow.cmake

@@ -57,6 +57,11 @@ find_library(ARROW_JEMALLOC_LIB_PATH NAMES arrow_jemalloc
   ${ARROW_SEARCH_LIB_PATH}
   NO_DEFAULT_PATH)
 
+find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
+  PATHS
+  ${ARROW_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
+
 if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_FOUND TRUE)
   set(ARROW_LIB_NAME libarrow)
@@ -75,7 +80,11 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_IPC_SHARED_LIB ${ARROW_LIBS}/${ARROW_IPC_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 
   set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_JEMALLOC_LIB_NAME}.a)
-  set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/${ARROW_JEMALLOC_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/${ARROW_JEMALLOC_LIB_NAME}${CMAKE_SHARED_LIBRARY
+_SUFFIX})
+
+  set(ARROW_PYTHON_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_PYTHON_LIB_NAME}.a)
+  set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 
   if (NOT Arrow_FIND_QUIETLY)
     message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}")

python/src/pyarrow/CMakeLists.txt

@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#######################################
-# Unit tests
-#######################################
-
-ADD_PYARROW_TEST(adapters/pandas-test)