Build arrow-cpp for each Python version

Change-Id: I4eb0fcb4b6648fa5947d511d4520dee940ae708c

python/CMakeLists.txt

@@ -227,21 +227,52 @@ find_package(Arrow REQUIRED)
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
 
 if (PYARROW_BUNDLE_ARROW_CPP)
+  # arrow
   configure_file(${ARROW_SHARED_LIB}
       ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX}
       COPYONLY)
+  configure_file(${ARROW_SHARED_LIB}.0
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX}.0
+      COPYONLY)
+  configure_file(${ARROW_SHARED_LIB}.0.0.0
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX}.0.0.0
+      COPYONLY)
   SET(ARROW_SHARED_LIB
       ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
+  # arrow_io
   configure_file(${ARROW_IO_SHARED_LIB}
       ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_io${CMAKE_SHARED_LIBRARY_SUFFIX}
       COPYONLY)
+  configure_file(${ARROW_IO_SHARED_LIB}.0
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_io${CMAKE_SHARED_LIBRARY_SUFFIX}.0
+      COPYONLY)
+  configure_file(${ARROW_IO_SHARED_LIB}.0.0.0
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_io${CMAKE_SHARED_LIBRARY_SUFFIX}.0.0.0
+      COPYONLY)
   SET(ARROW_IO_SHARED_LIB
       ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_io${CMAKE_SHARED_LIBRARY_SUFFIX})
+  # arrow_ipc
   configure_file(${ARROW_IPC_SHARED_LIB}
       ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_ipc${CMAKE_SHARED_LIBRARY_SUFFIX}
       COPYONLY)
+  configure_file(${ARROW_IPC_SHARED_LIB}.0
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_ipc${CMAKE_SHARED_LIBRARY_SUFFIX}.0
+      COPYONLY)
+  configure_file(${ARROW_IPC_SHARED_LIB}.0.0.0
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_ipc${CMAKE_SHARED_LIBRARY_SUFFIX}.0.0.0
+      COPYONLY)
   SET(ARROW_IPC_SHARED_LIB
       ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_ipc${CMAKE_SHARED_LIBRARY_SUFFIX})
+  configure_file(${ARROW_PYTHON_SHARED_LIB}
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX}
+      COPYONLY)
+  # arrow_python
+  configure_file(${ARROW_PYTHON_SHARED_LIB}.0
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX}.0
+      COPYONLY)
+  configure_file(${ARROW_PYTHON_SHARED_LIB}.0.0.0
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX}.0.0.0
+      COPYONLY)
   SET(ARROW_PYTHON_SHARED_LIB
       ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
 endif()
@@ -313,6 +344,12 @@ if (PYARROW_BUILD_JEMALLOC)
     configure_file(${ARROW_JEMALLOC_SHARED_LIB}
        ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}
        COPYONLY)
+    configure_file(${ARROW_JEMALLOC_SHARED_LIB}.0
+       ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}.0
+       COPYONLY)
+    configure_file(${ARROW_JEMALLOC_SHARED_LIB}.0.0.0
+       ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}.0.0.0
+       COPYONLY)
     SET(ARROW_JEMALLOC_SHARED_LIB
         ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
   endif()

python/cmake_modules/FindArrow.cmake

@@ -70,7 +70,7 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_JEMALLOC_LIB_NAME libarrow_jemalloc)
   set(ARROW_PYTHON_LIB_NAME libarrow_python)
 
-  set(ARROW_LIBS ${ARROW_SEARCH_LIB_PATH})
+  get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY)
   set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a)
   set(ARROW_SHARED_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 

python/manylinux1/Dockerfile-x86_64

@@ -52,11 +52,14 @@ RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
-RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF .
+RUN mkdir build-plain
+WORKDIR /arrow/cpp/build-plain
+RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF ..
 RUN make -j5 install
 
 WORKDIR /
-RUN git clone https://github.com/apache/parquet-cpp.git
+# RUN git clone https://github.com/apache/parquet-cpp.git
+RUN git clone https://github.com/xhochy/parquet-cpp.git
 WORKDIR /parquet-cpp
-RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF .
+RUN git checkout arrow_python-fixes && ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF .
 RUN make -j5 install

python/manylinux1/build_arrow.sh

@@ -52,6 +52,15 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     echo "=== (${PYTHON}) Installing build dependencies ==="
     $PIPI_IO "numpy==1.9.0"
     $PIPI_IO "cython==0.24"
+    $PIPI_IO "pandas==0.19.2"
+
+    echo "=== (${PYTHON}) Building Arrow C++ libraries ==="
+    ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON}
+    mkdir -p "${ARROW_BUILD_DIR}"
+    pushd "${ARROW_BUILD_DIR}"
+    PATH="$(cpython_path $PYTHON)/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON} ..
+    make -j5 install
+    popd
 
     # Clear output directory
     rm -rf dist/

python/setup.py

@@ -208,8 +208,16 @@ class build_ext(_build_ext):
         def move_lib(lib_name):
             lib_filename = (shared_library_prefix + lib_name +
                             shared_library_suffix)
+            print(lib_filename)
             shutil.move(pjoin(self.build_type, lib_filename),
                         pjoin(build_lib, 'pyarrow', lib_filename))
+            if lib_name.startswith("arrow"):
+                print(lib_filename + '.0.0.0')
+                shutil.move(pjoin(self.build_type, lib_filename + '.0.0.0'),
+                            pjoin(build_lib, 'pyarrow', lib_filename + '.0.0.0'))
+                print(lib_filename + '.0')
+                shutil.move(pjoin(self.build_type, lib_filename + '.0'),
+                            pjoin(build_lib, 'pyarrow', lib_filename + '.0'))
 
         if self.bundle_arrow_cpp:
             move_lib("arrow")