Try to modularize installing conda environment. Why does this not work?

Change-Id: Ic4b78546b705e326553f96661bdde72f252d19fe

dev/docker_common/install_pyarrow_conda_toolchain.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Miniconda - Python 3.6, 64-bit, x86, latest
+wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O conda.sh \
+ && /bin/bash conda.sh -b -p /opt/conda \
+ && rm conda.sh
+
+# create conda env with the required dependences
+conda create -y -q -c conda-forge -n pyarrow-dev \
+      python=$1 \
+      ipython \
+      matplotlib \
+      nomkl \
+      numpy \
+      six \
+      setuptools \
+      cython \
+      pandas \
+      pytest \
+      cmake \
+      flatbuffers \
+      rapidjson \
+      boost-cpp \
+      thrift-cpp \
+      snappy \
+      zlib \
+      gflags \
+      brotli \
+      jemalloc \
+      lz4-c \
+      zstd \
+      setuptools \
+      setuptools_scm
+ && conda clean --all

dev/gen_apidocs/Dockerfile

@@ -39,47 +39,24 @@ RUN apt-get update && \
         autoconf-archive \
         npm
 
-# This will install conda in /home/ubuntu/miniconda
-RUN wget -O /tmp/miniconda.sh \
-    https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
-    bash /tmp/miniconda.sh -b -p /home/ubuntu/miniconda && \
-    rm /tmp/miniconda.sh
-
-# Python dependencies
-# Create Conda environment
-RUN /home/ubuntu/miniconda/bin/conda create -y -q -n pyarrow-dev \
-        # Python
-        python=3.6 \
-        numpy \
-        pandas \
-        pytest \
-        cython \
-        ipython \
-        matplotlib \
-        numpydoc \
-        sphinx \
-        sphinx_bootstrap_theme \
-        six \
-        setuptools \
-        # C++
-        boost-cpp \
-        cmake \
-        flatbuffers \
-        rapidjson \
-        thrift-cpp \
-        snappy \
-        zlib \
-        brotli \
-        jemalloc \
-        lz4-c \
-        zstd \
-        doxygen \
-        maven \
-        -c conda-forge
 
 ENV CC=gcc-8
 ENV CXX=g++-8
 
 ADD . /apache-arrow
 WORKDIR /apache-arrow
+
+RUN arrow/dev/docker_common/install_pyarrow_conda_toolchain.sh 3.6
+ENV PATH="/opt/conda/bin:${PATH}"
+
+# Python dependencies
+# Create Conda environment
+RUN /home/ubuntu/miniconda/bin/conda create -y -q -n pyarrow-dev \
+        numpydoc \
+        sphinx \
+        sphinx_bootstrap_theme \
+        doxygen \
+        maven \
+        -c conda-forge
+
 CMD arrow/dev/gen_apidocs/create_documents.sh

dev/hdfs_integration/Dockerfile

@@ -18,8 +18,10 @@
 FROM gelog/hadoop
 
 ENV CC=gcc \
-    CXX=g++ \
-    PATH=/opt/conda/bin:$PATH
+    CXX=g++
+
+ADD . /apache-arrow
+WORKDIR /apache-arrow
 
 RUN apt-get update -y \
  && apt-get install -y \
@@ -30,34 +32,8 @@ RUN apt-get update -y \
 	  pkg-config \
 	  ninja-build
 
-# Miniconda - Python 3.6, 64-bit, x86, latest
-RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O conda.sh \
- && /bin/bash conda.sh -b -p /opt/conda \
- && rm conda.sh
-
-# create conda env with the required dependences
-RUN conda create -y -q -c conda-forge -n pyarrow-dev \
-      python=3.6 \
-      nomkl \
-      numpy \
-      six \
-      setuptools \
-      cython \
-      pandas \
-      pytest \
-      cmake \
-      flatbuffers \
-      rapidjson \
-      boost-cpp \
-      thrift-cpp \
-      snappy \
-      zlib \
-      gflags \
-      brotli \
-      jemalloc \
-      lz4-c \
-      zstd \
- && conda clean --all
+RUN arrow/dev/docker_common/install_pyarrow_conda_toolchain.sh 3.6
+ENV PATH="/opt/conda/bin:${PATH}"
 
 # installing in the previous step boost=1.60 and boost-cpp=1.67 gets installed,
 # cmake finds 1.60 and parquet fails to compile
@@ -69,7 +45,4 @@ RUN conda install -y -q -n pyarrow-dev -c conda-forge \
       libhdfs3=2.2.31 \
  && conda clean --all
 
-ADD . /apache-arrow
-WORKDIR /apache-arrow
-
 CMD arrow/dev/hdfs_integration/hdfs_integration.sh

dev/iwyu/Dockerfile

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 FROM ubuntu:18.04
+
 ADD . /apache-arrow
 WORKDIR /apache-arrow
 
@@ -29,6 +30,9 @@ RUN apt-get update -qq &&\
             libboost-filesystem-dev \
             software-properties-common
 
+CMD arrow/dev/docker_common/install_pyarrow_conda_toolchain.sh 3.6
+ENV PATH="/opt/conda/bin:${PATH}"
+
 RUN wget -O llvm-apt.gpg.key http://llvm.org/apt/llvm-snapshot.gpg.key &&\
     apt-key add llvm-apt.gpg.key &&\
     apt-add-repository -y \

dev/iwyu/run_iwyu.sh

@@ -52,6 +52,8 @@ popd
 # Add iwyu and iwyu_tool.py to path
 export PATH=$IWYU_BUILD_DIR/iwyu-build:$PATH
 
+source activate pyarrow-dev
+
 cmake -GNinja -DARROW_PYTHON=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
 
 # Make so that vendored bits are built

dev/spark_integration/Dockerfile

@@ -32,40 +32,13 @@ RUN apt-get update && \
     # Python dependencies
     apt-get install -y \
         pkg-config && \
-    # Create Conda environment
-    /home/ubuntu/miniconda/bin/conda create -y -q -n pyarrow-dev \
-        # Python
-        python=2.7 \
-        numpy \
-        pandas \
-        pytest \
-        cython \
-        ipython \
-        matplotlib \
-        six \
-        setuptools \
-        setuptools_scm \
-        # C++
-        boost-cpp \
-        cmake \
-        flatbuffers \
-        rapidjson \
-        thrift-cpp \
-        snappy \
-        zlib \
-        gflags \
-        brotli \
-        jemalloc \
-        lz4-c \
-        zstd \
-        -c conda-forge && \
-    /home/ubuntu/miniconda/bin/conda clean --all
-
-ENV PATH="/home/ubuntu/miniconda/bin:${PATH}"
 
 ADD . /apache-arrow
 WORKDIR /apache-arrow
 
+RUN arrow/dev/docker_common/install_pyarrow_conda_toolchain.sh 2.7
+ENV PATH="/opt/conda/bin:${PATH}"
+
 CMD arrow/dev/spark_integration/spark_integration.sh
 
 # BUILD: $ docker build -f arrow/dev/spark_integration/Dockerfile -t spark-arrow .