mirror of
https://github.com/apache/arrow.git
synced 2026-10-29 00:56:31 +08:00
ARROW-191: Python: Provide infrastructure for manylinux1 wheels
Change-Id: I5ee41524982f7f136ca718ed781f8608f0ffb2db
This commit is contained in:
Uwe L. Korn committed
1 parent
bf749f55a1
commit
278f8b0804
5 files changed
+206
No files matched your search
@@ -38,3 +38,6 @@ This product includes software from the CMake project
|
||||
* Copyright 2001-2009 Kitware, Inc.
|
||||
* Copyright 2012-2014 Continuum Analytics, Inc.
|
||||
* All rights reserved.
|
||||
|
||||
This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause)
|
||||
* Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved.
|
||||
@@ -0,0 +1,40 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License. See accompanying LICENSE file.
|
||||
|
||||
FROM arrow-base-x86_64
|
||||
|
||||
WORKDIR /
|
||||
ADD http://zlib.net/zlib-1.2.8.tar.gz /zlib-1.2.8.tar.gz
|
||||
RUN tar xf zlib-1.2.8.tar.gz
|
||||
WORKDIR zlib-1.2.8
|
||||
RUN CFLAGS=-fPIC cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr -DCMAKE_BUILD_TYPE=Release .
|
||||
RUN make -j5 install
|
||||
|
||||
WORKDIR /
|
||||
ADD https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz /snappy-1.1.3.tar.gz
|
||||
RUN tar xf snappy-1.1.3.tar.gz
|
||||
WORKDIR /snappy-1.1.3
|
||||
RUN ./configure --with-pic --prefix=/usr
|
||||
RUN make -j5 install
|
||||
|
||||
WORKDIR /
|
||||
ADD http://archive.apache.org/dist/thrift/0.9.1/thrift-0.9.1.tar.gz /thrift-0.9.1.tar.gz
|
||||
RUN tar xf thrift-0.9.1.tar.gz
|
||||
WORKDIR /thrift-0.9.1
|
||||
RUN ./configure LDFLAGS='-L/usr/lib64' CXXFLAGS='-fPIC' --without-qt4 --without-c_glib --without-csharp --without-java --without-erlang --without-nodejs --without-lua --without-python --without-perl --without-php --without-php_extension --without-ruby --without-haskell --without-go --without-d --without-tests --with-cpp --prefix=/usr --disable-shared --enable-static
|
||||
RUN make -j5 install
|
||||
|
||||
WORKDIR /
|
||||
RUN git clone https://github.com/apache/parquet-cpp.git
|
||||
WORKDIR /parquet-cpp
|
||||
RUN ARROW_HOME=/usr THRIFT_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON .
|
||||
RUN make -j5 install
|
||||
@@ -0,0 +1,47 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License. See accompanying LICENSE file.
|
||||
|
||||
FROM quay.io/pypa/manylinux1_x86_64:latest
|
||||
|
||||
# Install dependencies
|
||||
RUN yum install -y flex openssl-devel
|
||||
|
||||
WORKDIR /
|
||||
ADD http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz /boost_1_60_0.tar.gz
|
||||
RUN tar xf boost_1_60_0.tar.gz
|
||||
WORKDIR /boost_1_60_0
|
||||
RUN ./bootstrap.sh
|
||||
RUN ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system install
|
||||
|
||||
WORKDIR /
|
||||
ADD https://cmake.org/files/v3.5/cmake-3.5.2.tar.gz /cmake-3.5.2.tar.gz
|
||||
RUN tar xf cmake-3.5.2.tar.gz
|
||||
WORKDIR /cmake-3.5.2
|
||||
RUN ./configure --prefix=/usr
|
||||
RUN make -j5 install
|
||||
|
||||
WORKDIR /
|
||||
ADD https://github.com/google/flatbuffers/archive/v1.3.0.tar.gz /flatbuffers-1.3.0.tar.gz
|
||||
RUN tar xf flatbuffers-1.3.0.tar.gz
|
||||
WORKDIR /flatbuffers-1.3.0
|
||||
RUN CXXFLAGS='-fPIC' cmake -DFLATBUFFERS_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr .
|
||||
RUN make -j5 install
|
||||
|
||||
WORKDIR /
|
||||
RUN git clone https://github.com/matthew-brett/multibuild.git
|
||||
WORKDIR /multibuild
|
||||
RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
|
||||
|
||||
ADD arrow /arrow
|
||||
WORKDIR /arrow/cpp
|
||||
RUN FLATBUFFERS_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF .
|
||||
RUN make -j5 install
|
||||
@@ -0,0 +1,40 @@
|
||||
<!---
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. See accompanying LICENSE file.
|
||||
-->
|
||||
|
||||
## Manylinux1 wheels for Apache Arrow
|
||||
|
||||
This folder provides base Docker images and an infrastructure to build
|
||||
`manylinux1` compatible Python wheels that should be installable on all
|
||||
Linux distributions published in last four years.
|
||||
|
||||
The process is split up in two parts: There are base Docker images that build
|
||||
the native, Python-indenpendent dependencies. For these you can select if you
|
||||
want to also build the dependencies used for the Parquet support. Depending on
|
||||
these images, there is also a bash script that will build the pyarrow wheels
|
||||
for all supported Python versions and place them in the `dist` folder.
|
||||
|
||||
### Build instructions
|
||||
|
||||
```bash
|
||||
# Create a clean copy of the arrow source tree
|
||||
git clone ../../ arrow
|
||||
# Build the native baseimage
|
||||
docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
|
||||
# (optionally) build parquet-cpp
|
||||
docker build -t parquet_arrow-base-x86_64 -f Dockerfile-parquet_arrow-base-x86_64 .
|
||||
# Build the python packages
|
||||
docker run --rm -v $PWD:/io parquet_arrow-base-x86_64 /io/build_arrow.sh
|
||||
# Now the new packages are located in the dist/ folder
|
||||
ls -l dist/
|
||||
```
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License. See accompanying LICENSE file.
|
||||
#
|
||||
# Usage:
|
||||
# docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
|
||||
# or with Parquet support
|
||||
# docker run --rm -v $PWD:/io parquet_arrow-base-x86_64 /io/build_arrow.sh
|
||||
|
||||
# Build upon the scripts in https://github.com/matthew-brett/manylinux-builds
|
||||
# * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause)
|
||||
|
||||
PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7 3.4 3.5}"
|
||||
|
||||
# Package index with only manylinux1 builds
|
||||
MANYLINUX_URL=https://nipy.bic.berkeley.edu/manylinux
|
||||
|
||||
source /multibuild/manylinux_utils.sh
|
||||
|
||||
cd /arrow/python
|
||||
|
||||
# PyArrow build configuration
|
||||
export PYARROW_CMAKE_OPTIONS='-DCMAKE_BUILD_TYPE=Release'
|
||||
# Need as otherwise arrow_io is sometimes not linked
|
||||
export LDFLAGS="-Wl,--no-as-needed"
|
||||
export ARROW_HOME="/usr"
|
||||
|
||||
# Ensure the target directory exists
|
||||
mkdir -p /io/dist
|
||||
# Temporary directory to store the wheels that should be sent through auditwheel
|
||||
rm_mkdir unfixed_wheels
|
||||
|
||||
PY35_BIN=/opt/python/cp35-cp35m/bin
|
||||
$PY35_BIN/pip install 'pyelftools<0.24'
|
||||
$PY35_BIN/pip install 'git+https://github.com/xhochy/auditwheel.git@pyarrow-fixes'
|
||||
|
||||
# Override repair_wheelhouse function
|
||||
function repair_wheelhouse {
|
||||
local in_dir=$1
|
||||
local out_dir=$2
|
||||
for whl in $in_dir/*.whl; do
|
||||
if [[ $whl == *none-any.whl ]]; then
|
||||
cp $whl $out_dir
|
||||
else
|
||||
# Store libraries directly in . not .libs to fix problems with libpyarrow.so linkage.
|
||||
auditwheel -v repair -L . $whl -w $out_dir/
|
||||
fi
|
||||
done
|
||||
chmod -R a+rwX $out_dir
|
||||
}
|
||||
|
||||
for PYTHON in ${PYTHON_VERSIONS}; do
|
||||
PYTHON_INTERPRETER="$(cpython_path $PYTHON)/bin/python"
|
||||
PIP="$(cpython_path $PYTHON)/bin/pip"
|
||||
PIPI_IO="$PIP install -f $MANYLINUX_URL"
|
||||
PATH="$PATH:$(cpython_path $PYTHON)"
|
||||
|
||||
$PIPI_IO "numpy==1.9.0"
|
||||
$PIPI_IO "cython==0.24"
|
||||
|
||||
PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
|
||||
|
||||
rm_mkdir fixed_wheels
|
||||
repair_wheelhouse dist /io/dist
|
||||
done
|
||||
|
||||
Reference in new issue
Block a user