Cleanup, build pandas-test within main test suite

Change-Id: I291f4749c387e58a947b7df4c55f18de3c576d03

cpp/CMakeLists.txt

@@ -307,6 +307,14 @@ endfunction()
 #
 # Arguments after the test name will be passed to set_tests_properties().
 function(ADD_ARROW_TEST REL_TEST_NAME)
+  set(options)
+  set(single_value_args)
+  set(multi_value_args STATIC_LINK_LIBS)
+  cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
+  if(ARG_UNPARSED_ARGUMENTS)
+    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
   if(NO_TESTS OR NOT ARROW_BUILD_STATIC)
     return()
   endif()
@@ -316,7 +324,15 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
     # This test has a corresponding .cc file, set it up as an executable.
     set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
     add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
-    target_link_libraries(${TEST_NAME} ${ARROW_TEST_LINK_LIBS})
+
+    message(STATUS "link libs: ${STATIC_LINK_LIBS}")
+
+    if (ARG_STATIC_LINK_LIBS)
+      # Customize link libraries
+      target_link_libraries(${TEST_NAME} ${ARG_STATIC_LINK_LIBS})
+    else()
+      target_link_libraries(${TEST_NAME} ${ARROW_TEST_LINK_LIBS})
+    endif()
     add_dependencies(unittest ${TEST_NAME})
   else()
     # No executable, just invoke the test (probably a script) directly.
@@ -336,10 +352,6 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
       ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
   endif()
   set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest")
-
-  if(ARGN)
-    set_tests_properties(${TEST_NAME} PROPERTIES ${ARGN})
-  endif()
 endfunction()
 
 # A wrapper for add_dependencies() that is compatible with NO_TESTS.
@@ -802,8 +814,7 @@ if (${CLANG_FORMAT_FOUND})
   add_custom_target(format ${BUILD_SUPPORT_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 1
     `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h |
     sed -e '/_generated/g' |
-    sed -e '/windows_compatibility.h/g'`
-    `find ${CMAKE_CURRENT_SOURCE_DIR}/../python -name \\*.cc -or -name \\*.h`)
+    sed -e '/windows_compatibility.h/g'`)
 
   # runs clang format and exits with a non-zero exit code if any files need to be reformatted
   add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 0
@@ -864,8 +875,8 @@ endif()
 include(BuildUtils)
 
 ADD_ARROW_LIB(arrow
-    SOURCES ${ARROW_SRCS}
-    SHARED_LINK_FLAGS ${ARROW_SHARED_LINK_FLAGS}
+  SOURCES ${ARROW_SRCS}
+  SHARED_LINK_FLAGS ${ARROW_SHARED_LINK_FLAGS}
 )
 
 add_subdirectory(src/arrow)
@@ -879,6 +890,10 @@ endif()
 #----------------------------------------------------------------------
 # IPC library
 
+if(ARROW_PYTHON)
+  set(ARROW_IPC on)
+endif()
+
 ## Flatbuffers
 if(ARROW_IPC)
   if("$ENV{FLATBUFFERS_HOME}" STREQUAL "")

cpp/src/arrow/python/CMakeLists.txt

@@ -12,59 +12,19 @@
 #######################################
 # arrow_python
 #######################################
-# Add a new test case, with or without an executable that should be built.
-#
-# REL_TEST_NAME is the name of the test. It may be a single component
-# (e.g. monotime-test) or contain additional components (e.g.
-# net/net_util-test). Either way, the last component must be a globally
-# unique name.
-#
-# Arguments after the test name will be passed to set_tests_properties().
-function(ADD_ARROW_PYTHON_TEST REL_TEST_NAME)
-  if(NO_TESTS)
-    return()
-  endif()
-  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME}.cc)
-    # This test has a corresponding .cc file, set it up as an executable.
-    set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
-    add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
-    target_link_libraries(${TEST_NAME} ${PYARROW_TEST_LINK_LIBS})
-  else()
-    # No executable, just invoke the test (probably a script) directly.
-    set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
-  endif()
-
-  add_test(${TEST_NAME}
-    ${BUILD_SUPPORT_DIR}/run-test.sh ${TEST_PATH})
-  if(ARGN)
-    set_tests_properties(${TEST_NAME} PROPERTIES ${ARGN})
-  endif()
-endfunction()
-
-# A wrapper for add_dependencies() that is compatible with NO_TESTS.
-function(ADD_ARROW_PYTHON_TEST_DEPENDENCIES REL_TEST_NAME)
-  if(NO_TESTS)
-    return()
-  endif()
-  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
-  add_dependencies(${TEST_NAME} ${ARGN})
-endfunction()
 
 if (ARROW_BUILD_TESTS)
-  add_library(pyarrow_test_main STATIC
+  add_library(arrow_python_test_main STATIC
 	util/test_main.cc)
 
   if (APPLE)
-	target_link_libraries(pyarrow_test_main
+	target_link_libraries(arrow_python_test_main
       gtest
       dl)
-	set_target_properties(pyarrow_test_main
+	set_target_properties(arrow_python_test_main
       PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
   else()
-	target_link_libraries(pyarrow_test_main
+	target_link_libraries(arrow_python_test_main
       gtest
       pthread
       dl
@@ -72,6 +32,21 @@ if (ARROW_BUILD_TESTS)
   endif()
 endif()
 
+set(ARROW_PYTHON_MIN_TEST_LIBS
+  arrow_python_test_main
+  arrow_python_static
+  arrow_ipc_static
+  arrow_io_static
+  arrow_static)
+
+if(NOT APPLE AND ARROW_BUILD_TESTS)
+  ADD_THIRDPARTY_LIB(python
+    SHARED_LIB "${PYTHON_LIBRARIES}")
+  list(APPEND ARROW_PYTHON_MIN_TEST_LIBS python)
+endif()
+
+set(ARROW_PYTHON_TEST_LINK_LIBS ${ARROW_PYTHON_MIN_TEST_LIBS})
+
 # ----------------------------------------------------------------------
 
 set(ARROW_PYTHON_SRCS
@@ -89,10 +64,6 @@ set(ARROW_PYTHON_SHARED_LINK_LIBS
   arrow_shared
 )
 
-set(ARROW_PYTHON_TEST_LINK_LIBS
-  arrow_ipc_static
-  arrow_io_static)
-
 ADD_ARROW_LIB(arrow_python
   SOURCES ${ARROW_PYTHON_SRCS}
   SHARED_LINK_FLAGS ""
@@ -104,14 +75,19 @@ install(FILES
   api.h
   common.h
   config.h
+  do_import_numpy.h
   helper.h
   io.h
   numpy_interop.h
+  type_traits.h
   adapters/builtin.h
   adapters/pandas.h
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/python")
 
-# set_target_properties(pyarrow PROPERTIES
+# set_target_properties(arrow_python_shared PROPERTIES
 #     INSTALL_RPATH "\$ORIGIN")
 
-ADD_ARROW_PYTHON_TEST(adapters/pandas-test)
+if (ARROW_BUILD_TESTS)
+  ADD_ARROW_TEST(adapters/pandas-test
+    STATIC_LINK_LIBS "${ARROW_PYTHON_TEST_LINK_LIBS}")
+endif()

cpp/src/arrow/python/adapters/pandas-test.cc

@@ -24,17 +24,17 @@
 
 #include "arrow/array.h"
 #include "arrow/builder.h"
+#include "arrow/python/adapters/pandas.h"
 #include "arrow/schema.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
-#include "arrow/python/adapters/pandas.h"
 
 namespace arrow {
 namespace py {
 
 TEST(PandasConversionTest, TestObjectBlockWriteFails) {
-  StringBuilder builder;
+  StringBuilder builder(default_memory_pool());
   const char value[] = {'\xf1', '\0'};
 
   for (int i = 0; i < 1000; ++i) {

cpp/src/arrow/python/adapters/pandas.cc

@@ -947,7 +947,6 @@ inline Status ConvertListsLike(
   for (int c = 0; c < data.num_chunks(); c++) {
     auto arr = std::static_pointer_cast<ListArray>(data.chunk(c));
 
-    const uint8_t* data_ptr;
     const bool has_nulls = data.null_count() > 0;
     for (int64_t i = 0; i < arr->length(); ++i) {
       if (has_nulls && arr->IsNull(i)) {

cpp/src/arrow/python/common.cc

@@ -28,17 +28,17 @@ namespace arrow {
 namespace py {
 
 static std::mutex memory_pool_mutex;
-static MemoryPool* default_pyarrow_pool = nullptr;
+static MemoryPool* default_python_pool = nullptr;
 
 void set_default_memory_pool(MemoryPool* pool) {
   std::lock_guard<std::mutex> guard(memory_pool_mutex);
-  default_pyarrow_pool = pool;
+  default_python_pool = pool;
 }
 
 MemoryPool* get_memory_pool() {
   std::lock_guard<std::mutex> guard(memory_pool_mutex);
-  if (default_pyarrow_pool) {
-    return default_pyarrow_pool;
+  if (default_python_pool) {
+    return default_python_pool;
   } else {
     return default_memory_pool();
   }
@@ -47,22 +47,21 @@ MemoryPool* get_memory_pool() {
 // ----------------------------------------------------------------------
 // PyBuffer
 
-PyBuffer::PyBuffer(PyObject* obj)
-    : Buffer(nullptr, 0) {
-    if (PyObject_CheckBuffer(obj)) {
-        obj_ = PyMemoryView_FromObject(obj);
-        Py_buffer* buffer = PyMemoryView_GET_BUFFER(obj_);
-        data_ = reinterpret_cast<const uint8_t*>(buffer->buf);
-        size_ = buffer->len;
-        capacity_ = buffer->len;
-        is_mutable_ = false;
-        Py_INCREF(obj_);
-    }
+PyBuffer::PyBuffer(PyObject* obj) : Buffer(nullptr, 0) {
+  if (PyObject_CheckBuffer(obj)) {
+    obj_ = PyMemoryView_FromObject(obj);
+    Py_buffer* buffer = PyMemoryView_GET_BUFFER(obj_);
+    data_ = reinterpret_cast<const uint8_t*>(buffer->buf);
+    size_ = buffer->len;
+    capacity_ = buffer->len;
+    is_mutable_ = false;
+    Py_INCREF(obj_);
+  }
 }
 
 PyBuffer::~PyBuffer() {
-    PyAcquireGIL lock;
-    Py_DECREF(obj_);
+  PyAcquireGIL lock;
+  Py_DECREF(obj_);
 }
 
 }  // namespace py

cpp/src/arrow/python/config.cc

@@ -17,16 +17,16 @@
 
 #include <Python.h>
 
-#include "pyarrow/config.h"
+#include "arrow/python/config.h"
 
 namespace arrow {
 namespace py {
 
-void pyarrow_init() {}
+void Init() {}
 
 PyObject* numpy_nan = nullptr;
 
-void pyarrow_set_numpy_nan(PyObject* obj) {
+void set_numpy_nan(PyObject* obj) {
   Py_INCREF(obj);
   numpy_nan = obj;
 }

cpp/src/arrow/python/config.h

@@ -34,10 +34,10 @@ ARROW_EXPORT
 extern PyObject* numpy_nan;
 
 ARROW_EXPORT
-void pyarrow_init();
+void Init();
 
 ARROW_EXPORT
-void pyarrow_set_numpy_nan(PyObject* obj);
+void set_numpy_nan(PyObject* obj);
 
 }  // namespace py
 }  // namespace arrow

cpp/src/arrow/python/io.cc

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "pyarrow/io.h"
+#include "arrow/python/io.h"
 
 #include <cstdint>
 #include <cstdlib>
@@ -24,7 +24,7 @@
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
 
-#include "pyarrow/common.h"
+#include "arrow/python/common.h"
 
 namespace arrow {
 namespace py {

cpp/src/arrow/python/io.h

@@ -22,9 +22,9 @@
 #include "arrow/io/memory.h"
 #include "arrow/util/visibility.h"
 
-#include "pyarrow/config.h"
+#include "arrow/python/config.h"
 
-#include "pyarrow/common.h"
+#include "arrow/python/common.h"
 
 namespace arrow {
 

cpp/src/arrow/python/numpy_interop.h

@@ -34,7 +34,7 @@
 
 // This is required to be able to access the NumPy C API properly in C++ files
 // other than this main one
-#define PY_ARRAY_UNIQUE_SYMBOL pyarrow_ARRAY_API
+#define PY_ARRAY_UNIQUE_SYMBOL arrow_ARRAY_API
 #ifndef NUMPY_IMPORT_ARRAY
 #define NO_IMPORT_ARRAY
 #endif

cpp/src/arrow/python/type_traits.h

@@ -19,7 +19,7 @@
 
 #include <cstdint>
 
-#include "pyarrow/numpy_interop.h"
+#include "arrow/python/numpy_interop.h"
 
 #include "arrow/builder.h"
 #include "arrow/type.h"

cpp/src/arrow/python/util/CMakeLists.txt

@@ -16,21 +16,21 @@
 # under the License.
 
 #######################################
-# pyarrow_test_main
+# arrow/python_test_main
 #######################################
 
 if (PYARROW_BUILD_TESTS)
-  add_library(pyarrow_test_main STATIC
+  add_library(arrow/python_test_main STATIC
 	test_main.cc)
 
   if (APPLE)
-	target_link_libraries(pyarrow_test_main
+	target_link_libraries(arrow/python_test_main
       gtest
       dl)
-	set_target_properties(pyarrow_test_main
+	set_target_properties(arrow/python_test_main
       PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
   else()
-	target_link_libraries(pyarrow_test_main
+	target_link_libraries(arrow/python_test_main
       gtest
       pthread
       dl

cpp/src/arrow/python/util/test_main.cc

@@ -19,8 +19,8 @@
 
 #include <gtest/gtest.h>
 
-#include "pyarrow/do_import_numpy.h"
-#include "pyarrow/numpy_interop.h"
+#include "arrow/python/do_import_numpy.h"
+#include "arrow/python/numpy_interop.h"
 
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);

python/CMakeLists.txt

@@ -314,27 +314,6 @@ ADD_THIRDPARTY_LIB(arrow_io
 ADD_THIRDPARTY_LIB(arrow_ipc
   SHARED_LIB ${ARROW_IPC_SHARED_LIB})
 
-############################################################
-# Linker setup
-############################################################
-
-set(PYARROW_MIN_TEST_LIBS
-  pyarrow_test_main
-  pyarrow)
-
-set(PYARROW_MIN_TEST_LIBS
-  pyarrow_test_main
-  pyarrow
-  ${PYARROW_BASE_LIBS})
-
-if(NOT APPLE AND PYARROW_BUILD_TESTS)
-  ADD_THIRDPARTY_LIB(python
-    SHARED_LIB "${PYTHON_LIBRARIES}")
-  list(APPEND PYARROW_MIN_TEST_LIBS python)
-endif()
-
-set(PYARROW_TEST_LINK_LIBS ${PYARROW_MIN_TEST_LIBS})
-
 ############################################################
 # "make ctags" target
 ############################################################