mirror of
https://github.com/apache/arrow.git
synced 2026-10-29 00:56:31 +08:00
Fix up Dockerfiles to work again
Change-Id: Ib7280d478eb696323d21eb4f03f77f7e1e00f800
This commit is contained in:
Wes McKinney committed
1 parent
a487fc2e34
commit
873d6459cf
8 files changed
+126
-147
No files matched your search
@@ -123,6 +123,12 @@ bash dev/release/js-verify-release-candidate.sh 0.7.0 0
|
||||
```
|
||||
# Integration testing
|
||||
|
||||
Build the following base image used by multiple tests:
|
||||
|
||||
```shell
|
||||
docker build -t arrow_integration_xenial_base -f docker_common/Dockerfile.xenial.base .
|
||||
```
|
||||
|
||||
## HDFS C++ / Python support
|
||||
|
||||
```shell
|
||||
|
||||
@@ -14,74 +14,9 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
FROM ubuntu:14.04
|
||||
FROM arrow_integration_xenial_base
|
||||
|
||||
ADD . /apache-arrow
|
||||
WORKDIR /apache-arrow
|
||||
# Basic OS utilities
|
||||
RUN apt-get update && apt-get install -y \
|
||||
wget \
|
||||
git \
|
||||
gcc \
|
||||
g++
|
||||
# This will install conda in /home/ubuntu/miniconda
|
||||
RUN wget -O /tmp/miniconda.sh \
|
||||
https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
|
||||
bash /tmp/miniconda.sh -b -p /home/ubuntu/miniconda && \
|
||||
rm /tmp/miniconda.sh
|
||||
# Create Conda environment
|
||||
ENV PATH="/home/ubuntu/miniconda/bin:${PATH}"
|
||||
RUN conda create -y -q -n test-environment \
|
||||
python=3.6
|
||||
# Install dependencies
|
||||
RUN conda install -c conda-forge \
|
||||
numpy \
|
||||
pandas \
|
||||
bcolz \
|
||||
blosc \
|
||||
bokeh \
|
||||
boto3 \
|
||||
chest \
|
||||
cloudpickle \
|
||||
coverage \
|
||||
cytoolz \
|
||||
distributed \
|
||||
graphviz \
|
||||
h5py \
|
||||
ipython \
|
||||
partd \
|
||||
psutil \
|
||||
"pytest<=3.1.1" \
|
||||
scikit-image \
|
||||
scikit-learn \
|
||||
scipy \
|
||||
sqlalchemy \
|
||||
toolz
|
||||
# install pytables from defaults for now
|
||||
RUN conda install pytables
|
||||
|
||||
RUN pip install -q git+https://github.com/dask/partd --upgrade --no-deps
|
||||
RUN pip install -q git+https://github.com/dask/zict --upgrade --no-deps
|
||||
RUN pip install -q git+https://github.com/dask/distributed --upgrade --no-deps
|
||||
RUN pip install -q git+https://github.com/mrocklin/sparse --upgrade --no-deps
|
||||
RUN pip install -q git+https://github.com/dask/s3fs --upgrade --no-deps
|
||||
|
||||
RUN conda install -q -c conda-forge numba cython
|
||||
RUN pip install -q git+https://github.com/dask/fastparquet
|
||||
|
||||
RUN pip install -q \
|
||||
cachey \
|
||||
graphviz \
|
||||
moto \
|
||||
pyarrow \
|
||||
--upgrade --no-deps
|
||||
|
||||
RUN pip install -q \
|
||||
cityhash \
|
||||
flake8 \
|
||||
mmh3 \
|
||||
pandas_datareader \
|
||||
pytest-xdist \
|
||||
xxhash \
|
||||
pycodestyle
|
||||
|
||||
CMD arrow/dev/dask_integration/dask_integration.sh
|
||||
@@ -19,6 +19,55 @@
|
||||
# Set up environment and working directory
|
||||
cd /apache-arrow
|
||||
|
||||
source activate pyarrow-dev
|
||||
|
||||
# install pytables from defaults for now
|
||||
conda install -y pytables
|
||||
|
||||
pip install -q git+https://github.com/dask/partd --upgrade --no-deps
|
||||
pip install -q git+https://github.com/dask/zict --upgrade --no-deps
|
||||
pip install -q git+https://github.com/dask/distributed --upgrade --no-deps
|
||||
pip install -q git+https://github.com/mrocklin/sparse --upgrade --no-deps
|
||||
pip install -q git+https://github.com/dask/s3fs --upgrade --no-deps
|
||||
|
||||
conda install -y -q -c conda-forge numba cython \
|
||||
bcolz \
|
||||
blosc \
|
||||
bokeh \
|
||||
boto3 \
|
||||
chest \
|
||||
cloudpickle \
|
||||
coverage \
|
||||
cytoolz \
|
||||
distributed \
|
||||
graphviz \
|
||||
h5py \
|
||||
partd \
|
||||
psutil \
|
||||
"pytest<=3.1.1" \
|
||||
scikit-image \
|
||||
scikit-learn \
|
||||
sqlalchemy \
|
||||
toolz
|
||||
|
||||
pip install -q git+https://github.com/dask/fastparquet
|
||||
|
||||
pip install -q \
|
||||
cachey \
|
||||
graphviz \
|
||||
moto \
|
||||
pyarrow \
|
||||
--upgrade --no-deps
|
||||
|
||||
pip install -q \
|
||||
cityhash \
|
||||
flake8 \
|
||||
mmh3 \
|
||||
pandas_datareader \
|
||||
pytest-xdist \
|
||||
xxhash \
|
||||
pycodestyle
|
||||
|
||||
export ARROW_BUILD_TYPE=release
|
||||
export ARROW_HOME=$(pwd)/dist
|
||||
export PARQUET_HOME=$(pwd)/dist
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Miniconda - Python 3.6, 64-bit, x86, latest
|
||||
wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O conda.sh \
|
||||
&& /bin/bash conda.sh -b -p /opt/conda \
|
||||
&& rm conda.sh
|
||||
|
||||
# create conda env with the required dependences
|
||||
conda create -y -q -c conda-forge -n pyarrow-dev \
|
||||
python=$1 \
|
||||
ipython \
|
||||
matplotlib \
|
||||
nomkl \
|
||||
numpy \
|
||||
six \
|
||||
setuptools \
|
||||
cython \
|
||||
pandas \
|
||||
pytest \
|
||||
cmake \
|
||||
flatbuffers \
|
||||
rapidjson \
|
||||
boost-cpp \
|
||||
thrift-cpp \
|
||||
snappy \
|
||||
zlib \
|
||||
gflags \
|
||||
brotli \
|
||||
jemalloc \
|
||||
lz4-c \
|
||||
zstd \
|
||||
setuptools \
|
||||
setuptools_scm
|
||||
&& conda clean --all
|
||||
@@ -20,9 +20,6 @@ FROM gelog/hadoop
|
||||
ENV CC=gcc \
|
||||
CXX=g++
|
||||
|
||||
ADD . /apache-arrow
|
||||
WORKDIR /apache-arrow
|
||||
|
||||
RUN apt-get update -y \
|
||||
&& apt-get install -y \
|
||||
gcc \
|
||||
@@ -32,7 +29,37 @@ RUN apt-get update -y \
|
||||
pkg-config \
|
||||
ninja-build
|
||||
|
||||
RUN arrow/dev/docker_common/install_pyarrow_conda_toolchain.sh 3.6
|
||||
RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O conda.sh \
|
||||
&& /bin/bash conda.sh -b -p /opt/conda \
|
||||
&& rm conda.sh
|
||||
|
||||
RUN conda create -y -q -c conda-forge -n pyarrow-dev \
|
||||
python=$1 \
|
||||
ipython \
|
||||
matplotlib \
|
||||
nomkl \
|
||||
numpy \
|
||||
six \
|
||||
setuptools \
|
||||
cython \
|
||||
pandas \
|
||||
pytest \
|
||||
cmake \
|
||||
flatbuffers \
|
||||
rapidjson \
|
||||
boost-cpp \
|
||||
thrift-cpp \
|
||||
snappy \
|
||||
zlib \
|
||||
gflags \
|
||||
brotli \
|
||||
jemalloc \
|
||||
lz4-c \
|
||||
zstd \
|
||||
setuptools \
|
||||
setuptools_scm \
|
||||
&& conda clean --all
|
||||
|
||||
ENV PATH="/opt/conda/bin:${PATH}"
|
||||
|
||||
# installing in the previous step boost=1.60 and boost-cpp=1.67 gets installed,
|
||||
@@ -45,4 +72,7 @@ RUN conda install -y -q -n pyarrow-dev -c conda-forge \
|
||||
libhdfs3=2.2.31 \
|
||||
&& conda clean --all
|
||||
|
||||
ADD . /apache-arrow
|
||||
WORKDIR /apache-arrow
|
||||
|
||||
CMD arrow/dev/hdfs_integration/hdfs_integration.sh
|
||||
+2
-22
@@ -14,30 +14,10 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
FROM ubuntu:18.04
|
||||
|
||||
FROM arrow_integration_xenial_base
|
||||
|
||||
ADD . /apache-arrow
|
||||
WORKDIR /apache-arrow
|
||||
|
||||
RUN apt-get update -qq &&\
|
||||
apt-get install -qq \
|
||||
apt-utils \
|
||||
wget \
|
||||
gnupg \
|
||||
build-essential \
|
||||
ninja-build \
|
||||
libboost-system-dev \
|
||||
libboost-filesystem-dev \
|
||||
software-properties-common
|
||||
|
||||
CMD arrow/dev/docker_common/install_pyarrow_conda_toolchain.sh 3.6
|
||||
ENV PATH="/opt/conda/bin:${PATH}"
|
||||
|
||||
RUN wget -O llvm-apt.gpg.key http://llvm.org/apt/llvm-snapshot.gpg.key &&\
|
||||
apt-key add llvm-apt.gpg.key &&\
|
||||
apt-add-repository -y \
|
||||
"deb http://llvm.org/apt/bionic/ llvm-toolchain-bionic-6.0 main" &&\
|
||||
apt-get update -qq &&\
|
||||
apt-get install -q -y cmake clang-6.0 clang-format-6.0 clang-tidy-6.0 libclang-6.0-dev
|
||||
|
||||
CMD arrow/dev/iwyu/run_iwyu.sh
|
||||
@@ -45,6 +45,10 @@ export CXX=clang++-$CLANG_VERSION
|
||||
|
||||
mkdir -p iwyu-build
|
||||
pushd iwyu-build
|
||||
|
||||
# iwyu needs this
|
||||
apt-get install -y zlib1g-dev
|
||||
|
||||
cmake -G "Unix Makefiles" -DIWYU_LLVM_ROOT_PATH=/usr/lib/llvm-$CLANG_VERSION $IWYU_SRC
|
||||
make -j4
|
||||
popd
|
||||
|
||||
@@ -27,18 +27,43 @@ RUN apt-get update && \
|
||||
# install conda in /home/ubuntu/miniconda
|
||||
wget -O /tmp/miniconda.sh \
|
||||
https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
|
||||
bash /tmp/miniconda.sh -b -p /home/ubuntu/miniconda && \
|
||||
bash /tmp/miniconda.sh -b -p /opt/conda && \
|
||||
rm /tmp/miniconda.sh && \
|
||||
# Python dependencies
|
||||
apt-get install -y \
|
||||
pkg-config && \
|
||||
|
||||
ENV PATH="/opt/conda/bin:${PATH}"
|
||||
|
||||
RUN conda create -y -q -c conda-forge -n pyarrow-dev \
|
||||
python=2.7 \
|
||||
ipython \
|
||||
nomkl \
|
||||
numpy \
|
||||
six \
|
||||
setuptools \
|
||||
cython \
|
||||
pandas \
|
||||
pytest \
|
||||
cmake \
|
||||
flatbuffers \
|
||||
rapidjson \
|
||||
boost-cpp \
|
||||
thrift-cpp \
|
||||
snappy \
|
||||
zlib \
|
||||
gflags \
|
||||
brotli \
|
||||
jemalloc \
|
||||
lz4-c \
|
||||
zstd \
|
||||
setuptools \
|
||||
setuptools_scm \
|
||||
&& conda clean --all
|
||||
|
||||
ADD . /apache-arrow
|
||||
WORKDIR /apache-arrow
|
||||
|
||||
RUN arrow/dev/docker_common/install_pyarrow_conda_toolchain.sh 2.7
|
||||
ENV PATH="/opt/conda/bin:${PATH}"
|
||||
|
||||
CMD arrow/dev/spark_integration/spark_integration.sh
|
||||
|
||||
# BUILD: $ docker build -f arrow/dev/spark_integration/Dockerfile -t spark-arrow .
|
||||
|
||||
Reference in new issue
Block a user