Use a default conda environment other than base

ci/docker/conda-cpp.dockerfile

@@ -38,22 +38,26 @@ COPY ci/scripts/install_conda.sh \
 RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix}
 RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix}
 
-# create a test environment, to shield from base environment and avoid conflicts
-RUN conda create -n testenv
-
 # install the required conda packages into the test environment
 COPY ci/conda_env_cpp.yml \
      ci/conda_env_gandiva.yml \
      ci/conda_env_unix.yml \
      /arrow/ci/
-RUN conda install -n testenv -q \
+RUN conda create -n arrow -q \
         --file arrow/ci/conda_env_unix.yml \
         --file arrow/ci/conda_env_cpp.yml \
         --file arrow/ci/conda_env_gandiva.yml \
         git compilers && \
     conda clean --all
 
-ENV CONDA_PREFIX=${prefix}/envs/testenv
+# activate the created environment by default
+RUN echo "conda activate arrow" >> ~/.profile
+
+# use login shell to activate arrow environment un the RUN commands
+SHELL [ "/bin/bash", "-c", "-l" ]
+
+# use login shell when running the container
+ENTRYPOINT [ "/bin/bash", "-c", "-l" ]
 
 ENV ARROW_S3=ON \
     ARROW_ORC=ON \

ci/docker/conda-integration.dockerfile

@@ -25,7 +25,7 @@ ARG node=11
 ARG jdk=8
 ARG go=1.12
 
-RUN conda install -n testenv -q \
+RUN conda install -q \
         numpy \
         maven=${maven} \
         nodejs=${node} \

ci/docker/conda-python-hdfs.dockerfile

@@ -22,7 +22,7 @@ FROM ${org}/${arch}-conda-python-${python}:latest
 
 ARG jdk=8
 ARG maven=3.5
-RUN conda install -n testenv -q \
+RUN conda install -q \
         pandas \
         openjdk=${jdk} \
         maven=${maven} && \

ci/docker/conda-python-spark.dockerfile

@@ -22,7 +22,7 @@ FROM ${org}/${arch}-conda-python-${python}:latest
 
 ARG jdk=8
 ARG maven=3.5
-RUN conda install -n testenv -q \
+RUN conda install -q \
         patch \
         pandas \
         openjdk=${jdk} \
@@ -31,10 +31,9 @@ RUN conda install -n testenv -q \
 
 # installing specific version of spark
 ARG spark=master
-COPY ci/scripts/install_spark.sh \
-    ci/etc/ARROW-6429.patch \
-    /arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_spark.sh ${spark}
+COPY ci/scripts/install_spark.sh /arrow/ci/scripts/
+COPY ci/etc/ARROW-6429.patch /arrow/ci/etc/
+RUN /arrow/ci/scripts/install_spark.sh ${spark} /spark /arrow/ci/etc
 
 # build cpp with tests
 ENV CC=gcc \

ci/docker/conda-python-turbodbc.dockerfile

@@ -30,7 +30,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     rm -rf /var/lib/apt/lists/*
 
 # install turbodbc dependencies from conda-forge
-RUN conda install -n testenv -c conda-forge -q\
+RUN conda install -c conda-forge -q\
         pybind11 \
         pytest-cov \
         mock \
@@ -45,6 +45,6 @@ RUN service postgresql start && \
 
 ARG turbodbc=latest
 COPY ci/scripts/install_turbodbc.sh /arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_turbodbc.sh ${turbodbc}
+RUN /arrow/ci/scripts/install_turbodbc.sh ${turbodbc} /turbodbc
 
 ENV TURBODBC_TEST_CONFIGURATION_FILES "query_fixtures_postgresql.json"

ci/docker/conda-python.dockerfile

@@ -22,18 +22,16 @@ FROM ${org}/${arch}-conda-cpp:latest
 # install python specific packages
 ARG python=3.6
 COPY ci/conda_env_python.yml /arrow/ci/
-RUN conda install -n testenv -q \
+RUN conda install -q \
         --file arrow/ci/conda_env_python.yml \
+        $([ "$python" == "2.7" ] && echo "futures enum34") \
         python=${python} \
         nomkl && \
     conda clean --all
 
-COPY python/requirements.txt /arrow/python/
-RUN bash -c "source activate testenv && pip install -r /arrow/python/requirements.txt"
-
 ENV ARROW_PYTHON=ON \
     ARROW_BUILD_STATIC=OFF \
     ARROW_BUILD_TESTS=OFF \
     ARROW_BUILD_UTILITIES=OFF \
     ARROW_TENSORFLOW=ON \
-    ARROW_USE_GLOG=OFF \
+    ARROW_USE_GLOG=OFF

ci/docker/conda-r.dockerfile

@@ -31,8 +31,7 @@ RUN conda install -n testenv -q \
 # Ensure parallel compilation of each individual package
 RUN printf "\nMAKEFLAGS=-j8\n" >> /opt/conda/envs/testenv/lib/R/etc/Makeconf
 
-ENV \
-    ARROW_BUILD_STATIC=OFF \
+ENV ARROW_BUILD_STATIC=OFF \
     ARROW_BUILD_TESTS=OFF \
     ARROW_BUILD_UTILITIES=OFF \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \

ci/scripts/install_conda.sh

@@ -53,7 +53,6 @@ rm /tmp/miniconda.sh
 
 # Like "conda init", but for POSIX sh rather than bash
 ln -s ${prefix}/etc/profile.d/conda.sh /etc/profile.d/conda.sh
-echo "conda activate base" >> ~/.profile
 
 # Configure
 source /etc/profile.d/conda.sh

ci/scripts/install_dask.sh

@@ -26,8 +26,6 @@ fi
 
 dask=$1
 
-source activate testenv
-
 if [ "${dask}" = "master" ]; then
   pip install https://github.com/dask/dask/archive/master.tar.gz#egg=dask[dataframe]
 elif [ "${dask}" = "latest" ]; then
@@ -35,5 +33,4 @@ elif [ "${dask}" = "latest" ]; then
 else
   conda install -q dask=${dask}
 fi
-
 conda clean --all

ci/scripts/install_pandas.sh

@@ -26,8 +26,6 @@ fi
 
 pandas=$1
 
-source activate testenv
-
 if [ "${pandas}" = "master" ]; then
   conda install -q numpy
   pip install https://github.com/pandas-dev/pandas/archive/master.tar.gz

ci/scripts/install_spark.sh

@@ -20,17 +20,16 @@
 set -e
 
 if [ "$#" -ne 1 ]; then
-  echo "Usage: $0 <spark version>"
+  echo "Usage: $0 <spark version> <target directory> <patches directory>"
   exit 1
 fi
 
 spark=$1
+target=$2
+patches=${3}
 
-source activate testenv
+mkdir "${target}"
+wget -q -O - https://github.com/apache/spark/archive/${spark}.tar.gz | tar -xzf - --strip-components=1 -C "${target}"
 
-mkdir /spark
-wget -q -O - https://github.com/apache/spark/archive/${spark}.tar.gz | tar -xzf - --strip-components=1 -C /spark
-
-# patch spark to build with current Arrow Java
-# patch -d /spark -p1 -i /arrow/ci/scripts/ARROW-6429.patch
-rm /arrow/ci/scripts/ARROW-6429.patch
+# Patch spark to build with current Arrow Java
+# patch -d "${target}" -p1 -i "${patches}/ARROW-6429.patch"

ci/scripts/install_turbodbc.sh

@@ -20,21 +20,18 @@
 set -e
 
 if [ "$#" -ne 1 ]; then
-  echo "Usage: $0 <turbodbc version>"
+  echo "Usage: $0 <turbodbc version> <target directory>"
   exit 1
 fi
 
 turbodbc=$1
+target=$2
 
-source activate testenv
-
-git clone --recurse-submodules https://github.com/blue-yonder/turbodbc /turbodbc
+git clone --recurse-submodules https://github.com/blue-yonder/turbodbc "${target}"
 if [ "${turbodbc}" = "master" ]; then
-  git -C /turbodbc checkout master;
+  git -C "${target}" checkout master;
 elif [ "${turbodbc}" = "latest" ]; then
-  git -C /turbodbc checkout $(git describe --tags);
+  git -C "${target}" checkout $(git describe --tags);
 else
-  git -C /turbodbc checkout ${turbodbc};
+  git -C "${target}" checkout ${turbodbc};
 fi
-
-conda clean --all

docker-compose.yml

@@ -76,10 +76,8 @@ services:
     volumes: &conda-volumes
       - .:/arrow:delegated
       - ${ARCH}-conda-cache:/build:delegated
-    entrypoint: &conda-entrypoint /bin/bash -c -l
     command: &cpp-conda-command
-      ["conda activate testenv &&
-        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/cpp_test.sh /arrow /build/cpp"]
 
   cuda-cpp:
@@ -288,10 +286,8 @@ services:
         python: ${PYTHON}
     shm_size: *shm-size
     volumes: *conda-volumes
-    entrypoint: *conda-entrypoint
     command: &python-conda-command
-      ["conda activate testenv &&
-        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/python_build.sh /arrow /build/python &&
         /arrow/ci/scripts/python_test.sh /arrow"]
 
@@ -428,7 +424,6 @@ services:
         pandas: ${PANDAS}
     shm_size: *shm-size
     volumes: *conda-volumes
-    entrypoint: *conda-entrypoint
     command: *python-conda-command
 
   conda-python-dask:
@@ -454,10 +449,8 @@ services:
         dask: ${DASK}
     shm_size: *shm-size
     volumes: *conda-volumes
-    entrypoint: *conda-entrypoint
     command:
-      ["conda activate testenv &&
-        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/python_build.sh /arrow /build/python &&
         /arrow/ci/scripts/integration_dask.sh"]
 
@@ -484,10 +477,8 @@ services:
         turbodbc: ${TURBODBC}
     shm_size: *shm-size
     volumes: *conda-volumes
-    entrypoint: *conda-entrypoint
     command:
-      ["conda activate testenv &&
-        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/python_build.sh /arrow /build/python &&
         /arrow/ci/scripts/integration_turbodbc.sh /turbodbc /build/turbodbc"]
 
@@ -546,10 +537,8 @@ services:
         r: ${R}
     shm_size: *shm-size
     volumes: *conda-volumes
-    entrypoint: *conda-entrypoint
     command:
-      ["conda activate testenv &&
-        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/r_build.sh /arrow &&
         /arrow/ci/scripts/r_test.sh /arrow"]
 
@@ -759,13 +748,11 @@ services:
         node: ${NODE}
         go: ${GO}
     volumes: *conda-volumes
-    entrypoint: *conda-entrypoint
     environment:
       # tell archery where the arrow binaries are located
       ARROW_CPP_EXE_PATH: /build/cpp/debug
     command:
-      ["conda activate testenv &&
-        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/go_build.sh /arrow &&
         /arrow/ci/scripts/java_build.sh /arrow &&
         /arrow/ci/scripts/js_build.sh /arrow &&
@@ -892,10 +879,8 @@ services:
       - ARROW_HIVESERVER2_TEST_HOST=impala
     shm_size: *shm-size
     volumes: *conda-volumes
-    entrypoint: *conda-entrypoint
     command:
-      ["conda activate testenv &&
-        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/integration_hiveserver2.sh /arrow /build/cpp"]
 
   conda-python-hdfs:
@@ -930,10 +915,8 @@ services:
       - .:/arrow:delegated
       - maven-cache:/root/.m2:delegated
       - ${ARCH}-conda-cache:/build:delegated
-    entrypoint: *conda-entrypoint
     command:
-      ["conda activate testenv &&
-        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/python_build.sh /arrow /build/python &&
         /arrow/ci/scripts/integration_hdfs.sh /arrow /build/cpp"]
 
@@ -960,10 +943,8 @@ services:
         spark: ${SPARK}
     shm_size: *shm-size
     volumes: *conda-maven-volumes
-    entrypoint: *conda-entrypoint
     command:
-      ["conda activate testenv &&
-        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/python_build.sh /arrow /build/python &&
         /arrow/ci/scripts/java_build.sh /arrow &&
         /arrow/ci/scripts/integration_spark.sh /arrow /spark"]