ARROW-6954: [Python] [CI] Add Python 3.8 to CI matrix

Also use a separate conda environment rather than installing everything
in the base conda environment, as conda can complain about conflicts.

.github/workflows/python.yml

@@ -71,7 +71,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: [2.7, 3.6]
+        python: [2.7, 3.6, 3.8]
     env:
       PYTHON: ${{ matrix.python }}
       PANDAS: latest

ci/conda_env_python.yml

@@ -16,7 +16,7 @@
 # under the License.
 
 # don't add pandas here, because it is not a mandatory test dependency
-cython=0.29.7
+cython
 cloudpickle
 hypothesis
 numpy>=1.14
@@ -25,4 +25,4 @@ pytest-faulthandler
 pytest-lazy-fixture
 pytz
 setuptools
-setuptools_scm=3.2.0
+setuptools_scm

ci/conda_env_unix.yml

@@ -22,4 +22,3 @@ ccache
 orc
 pkg-config
 rsync
-valgrind

ci/docker/conda-cpp.dockerfile

@@ -29,27 +29,32 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-ENV PATH=${prefix}/bin:$PATH \
-    CONDA_PREFIX=${prefix}
+ENV PATH=${prefix}/bin:$PATH
+
 # install conda and minio
 COPY ci/scripts/install_conda.sh \
      ci/scripts/install_minio.sh \
      /arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix} && \
-    /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix}
+RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix}
+RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix}
 
-# install the required conda packages
+# create a test environment, to shield from base environment and avoid conflicts
+RUN conda create -n testenv
+
+# install the required conda packages into the test environment
 COPY ci/conda_env_cpp.yml \
      ci/conda_env_gandiva.yml \
      ci/conda_env_unix.yml \
      /arrow/ci/
-RUN conda install -q \
+RUN conda install -n testenv -q \
         --file arrow/ci/conda_env_unix.yml \
         --file arrow/ci/conda_env_cpp.yml \
         --file arrow/ci/conda_env_gandiva.yml \
         git compilers && \
     conda clean --all
 
+ENV CONDA_PREFIX=${prefix}/envs/testenv
+
 ENV ARROW_S3=ON \
     ARROW_ORC=ON \
     ARROW_GANDIVA=ON \

ci/docker/conda-integration.dockerfile

@@ -25,7 +25,7 @@ ARG node=11
 ARG jdk=8
 ARG go=1.12
 
-RUN conda install -q \
+RUN conda install -n testenv -q \
         numpy \
         maven=${maven} \
         nodejs=${node} \

ci/docker/conda-python-dask.dockerfile

@@ -21,10 +21,5 @@ ARG python=3.6
 FROM ${org}/${arch}-conda-python-${python}:latest
 
 ARG dask=latest
-RUN if [ "${dask}" = "master" ]; then \
-        pip install git+https://github.com/dask/dask; \
-    elif [ "${dask}" = "latest" ]; then \
-        conda install -q dask && conda clean --all; \
-    else \
-        conda install -q dask=${dask} && conda clean --all; \
-    fi
+COPY ci/scripts/install_dask.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_dask.sh ${dask}

ci/docker/conda-python-hdfs.dockerfile

@@ -22,7 +22,7 @@ FROM ${org}/${arch}-conda-python-${python}:latest
 
 ARG jdk=8
 ARG maven=3.5
-RUN conda install -q \
+RUN conda install -n testenv -q \
         pandas \
         openjdk=${jdk} \
         maven=${maven} && \
@@ -43,7 +43,7 @@ ENV CC=gcc \
     ARROW_GANDIVA=OFF \
     ARROW_PLASMA=OFF \
     ARROW_PARQUET=ON \
-    ARROW_ORC=ON \
+    ARROW_ORC=OFF \
     ARROW_HDFS=ON \
     ARROW_PYTHON=ON \
     ARROW_BUILD_TESTS=ON

ci/docker/conda-python-pandas.dockerfile

@@ -21,10 +21,5 @@ ARG python=3.6
 FROM ${org}/${arch}-conda-python-${python}:latest
 
 ARG pandas=latest
-RUN if [ "${pandas}" = "master" ]; then \
-        pip install git+https://github.com/pandas-dev/pandas; \
-    elif [ "${pandas}" = "latest" ]; then \
-        conda install -q pandas && conda clean --all; \
-    else \
-        conda install -q pandas=${pandas} && conda clean --all; \
-    fi
+COPY ci/scripts/install_pandas.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_pandas.sh ${pandas}

ci/docker/conda-python-spark.dockerfile

@@ -22,7 +22,7 @@ FROM ${org}/${arch}-conda-python-${python}:latest
 
 ARG jdk=8
 ARG maven=3.5
-RUN conda install -q \
+RUN conda install -n testenv -q \
         patch \
         pandas \
         openjdk=${jdk} \
@@ -31,11 +31,10 @@ RUN conda install -q \
 
 # installing specific version of spark
 ARG spark=master
-RUN mkdir /spark && wget -q -O - https://github.com/apache/spark/archive/${spark}.tar.gz | tar -xzf - --strip-components=1 -C /spark
-# patch spark to build with current Arrow Java
-COPY ci/etc/ARROW-6429.patch /tmp/
-RUN patch -d /spark -p1 -i /tmp/ARROW-6429.patch && \
-    rm /tmp/ARROW-6429.patch
+COPY ci/scripts/install_spark.sh \
+    ci/etc/ARROW-6429.patch \
+    /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_spark.sh ${spark}
 
 # build cpp with tests
 ENV CC=gcc \

ci/docker/conda-python-turbodbc.dockerfile

@@ -30,7 +30,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     rm -rf /var/lib/apt/lists/*
 
 # install turbodbc dependencies from conda-forge
-RUN conda install -c conda-forge \
+RUN conda install -n testenv -c conda-forge -q\
         pybind11 \
         pytest-cov \
         mock \
@@ -44,13 +44,7 @@ RUN service postgresql start && \
         "ALTER USER postgres WITH PASSWORD 'password';"
 
 ARG turbodbc=latest
-RUN git clone --recurse-submodules https://github.com/blue-yonder/turbodbc /turbodbc && \
-    if [ "${turbodbc}" = "master" ]; then \
-        git -C /turbodbc checkout master; \
-    elif [ "${turbodbc}" = "latest" ]; then \
-        git -C /turbodbc checkout $(git describe --tags); \
-    else \
-        git -C /turbodbc checkout ${turbodbc}; \
-    fi
+COPY ci/scripts/install_turbodbc.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_turbodbc.sh ${turbodbc}
 
 ENV TURBODBC_TEST_CONFIGURATION_FILES "query_fixtures_postgresql.json"

ci/docker/conda-python.dockerfile

@@ -22,13 +22,15 @@ FROM ${org}/${arch}-conda-cpp:latest
 # install python specific packages
 ARG python=3.6
 COPY ci/conda_env_python.yml /arrow/ci/
-RUN conda install -q \
+RUN conda install -n testenv -q \
         --file arrow/ci/conda_env_python.yml \
         python=${python} \
-        tensorflow \
         nomkl && \
     conda clean --all
 
+COPY python/requirements.txt /arrow/python/
+RUN bash -c "source activate testenv && pip install -r /arrow/python/requirements.txt"
+
 ENV ARROW_PYTHON=ON \
     ARROW_BUILD_STATIC=OFF \
     ARROW_BUILD_TESTS=OFF \

ci/docker/conda-r.dockerfile

@@ -22,20 +22,21 @@ FROM ${org}/${arch}-conda-cpp:latest
 # install R specific packages
 ARG r=3.6.1
 COPY ci/conda_env_r.yml /arrow/ci/
-RUN conda install -q \
+RUN conda install -n testenv -q \
         --file arrow/ci/conda_env_r.yml \
         r-base=$r \
         nomkl && \
     conda clean --all
 
 # Ensure parallel compilation of each individual package
-RUN printf "\nMAKEFLAGS=-j8\n" >> /opt/conda/lib/R/etc/Makeconf
+RUN printf "\nMAKEFLAGS=-j8\n" >> /opt/conda/envs/testenv/lib/R/etc/Makeconf
 
 ENV \
     ARROW_BUILD_STATIC=OFF \
     ARROW_BUILD_TESTS=OFF \
     ARROW_BUILD_UTILITIES=OFF \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \
+    ARROW_FLIGHT=OFF \
     ARROW_GANDIVA=OFF \
     ARROW_ORC=OFF \
     ARROW_PARQUET=ON \

ci/docker/linux-apt-r.dockerfile

@@ -64,6 +64,7 @@ ENV \
     ARROW_BUILD_TESTS=OFF \
     ARROW_BUILD_UTILITIES=OFF \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \
+    ARROW_FLIGHT=OFF \
     ARROW_GANDIVA=OFF \
     ARROW_ORC=OFF \
     ARROW_PARQUET=ON \

ci/scripts/install_conda.sh

@@ -51,6 +51,7 @@ wget -nv https://repo.continuum.io/miniconda/Miniconda3-${version}-${platform}-$
 bash /tmp/miniconda.sh -b -p ${prefix}
 rm /tmp/miniconda.sh
 
+# Like "conda init", but for POSIX sh rather than bash
 ln -s ${prefix}/etc/profile.d/conda.sh /etc/profile.d/conda.sh
 echo "conda activate base" >> ~/.profile
 

ci/scripts/install_dask.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 <dask version>"
+  exit 1
+fi
+
+dask=$1
+
+source activate testenv
+
+if [ "${dask}" = "master" ]; then
+  pip install https://github.com/dask/dask/archive/master.tar.gz#egg=dask[dataframe]
+elif [ "${dask}" = "latest" ]; then
+  conda install -q dask
+else
+  conda install -q dask=${dask}
+fi
+
+conda clean --all

ci/scripts/install_pandas.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 <pandas version>"
+  exit 1
+fi
+
+pandas=$1
+
+source activate testenv
+
+if [ "${pandas}" = "master" ]; then
+  conda install -q numpy
+  pip install https://github.com/pandas-dev/pandas/archive/master.tar.gz
+elif [ "${pandas}" = "latest" ]; then
+  conda install -q pandas
+else
+  conda install -q pandas=${pandas}
+fi
+
+conda clean --all

ci/scripts/install_spark.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 <spark version>"
+  exit 1
+fi
+
+spark=$1
+
+source activate testenv
+
+mkdir /spark
+wget -q -O - https://github.com/apache/spark/archive/${spark}.tar.gz | tar -xzf - --strip-components=1 -C /spark
+
+# patch spark to build with current Arrow Java
+# patch -d /spark -p1 -i /arrow/ci/scripts/ARROW-6429.patch
+rm /arrow/ci/scripts/ARROW-6429.patch

ci/scripts/install_turbodbc.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 <turbodbc version>"
+  exit 1
+fi
+
+turbodbc=$1
+
+source activate testenv
+
+git clone --recurse-submodules https://github.com/blue-yonder/turbodbc /turbodbc
+if [ "${turbodbc}" = "master" ]; then
+  git -C /turbodbc checkout master;
+elif [ "${turbodbc}" = "latest" ]; then
+  git -C /turbodbc checkout $(git describe --tags);
+else
+  git -C /turbodbc checkout ${turbodbc};
+fi
+
+conda clean --all

ci/scripts/integration_dask.sh

@@ -24,6 +24,9 @@ set -e
 python -c "import pyarrow.orc"
 python -c "import pyarrow.parquet"
 
+# check that dask.dataframe is correctly installed
+python -c "import dask.dataframe"
+
 # TODO(kszucs): the following tests are also uses pyarrow
 # pytest -sv --pyargs dask.bytes.tests.test_s3
 # pytest -sv --pyargs dask.bytes.tests.test_hdfs

ci/scripts/python_build.sh

@@ -39,11 +39,7 @@ pushd ${source_dir}
 relative_build_dir=$(realpath --relative-to=. $build_dir)
 
 # not nice, but prevents mutating the mounted the source directory for docker
-python setup.py build_ext --build-temp $relative_build_dir \
-                        --build-lib $relative_build_dir \
-                build_py --build-lib $relative_build_dir \
-                egg_info --egg-base $relative_build_dir \
-                install_lib --build-dir $relative_build_dir \
+python setup.py build --build-base $build_dir \
                 install --single-version-externally-managed \
                         --record $relative_build_dir/record.txt
 

dev/archery/setup.py

@@ -32,7 +32,7 @@ setup(
     maintainer='Arrow Developers',
     maintainer_email='dev@arrow.apache.org',
     packages=['archery'],
-    install_requires=['click'],
+    install_requires=['click', 'six'],
     entry_points='''
         [console_scripts]
         archery=archery.cli:archery

dev/tasks/tasks.yml

@@ -97,6 +97,7 @@ groups:
     - test-conda-python-2.7
     - test-conda-python-3.6
     - test-conda-python-3.7
+    - test-conda-python-3.8
     - test-debian-10-python-3
     - test-ubuntu-18.04-python-3
     - test-fedora-29-python-3
@@ -111,8 +112,9 @@ groups:
     - test-conda-python-2.7-pandas-master
     - test-conda-python-3.7-pandas-latest
     - test-conda-python-3.7-pandas-master
+    - test-conda-python-3.8-pandas-latest
     - test-conda-python-3.7-dask-latest
-    - test-conda-python-3.7-dask-master
+    - test-conda-python-3.8-dask-master
     - test-conda-python-3.7-turbodbc-latest
     - test-conda-python-3.7-turbodbc-master
     - test-conda-python-3.7-hdfs-2.9.2
@@ -136,6 +138,7 @@ groups:
     - test-conda-python-2.7
     - test-conda-python-3.6
     - test-conda-python-3.7
+    - test-conda-python-3.8
     - test-debian-10-python-3
     - test-ubuntu-18.04-python-3
     - test-fedora-29-python-3
@@ -165,6 +168,7 @@ groups:
     - test-conda-python-2.7
     - test-conda-python-3.6
     - test-conda-python-3.7
+    - test-conda-python-3.8
     - test-debian-10-python-3
     - test-ubuntu-18.04-python-3
     - test-fedora-29-python-3
@@ -183,8 +187,9 @@ groups:
     - test-conda-python-2.7-pandas-master
     - test-conda-python-3.7-pandas-latest
     - test-conda-python-3.7-pandas-master
+    - test-conda-python-3.8-pandas-latest
     - test-conda-python-3.7-dask-latest
-    - test-conda-python-3.7-dask-master
+    - test-conda-python-3.8-dask-master
     - test-conda-python-3.7-turbodbc-latest
     - test-conda-python-3.7-turbodbc-master
     - test-conda-python-3.7-hdfs-2.9.2
@@ -260,8 +265,9 @@ groups:
     - test-conda-python-2.7-pandas-master
     - test-conda-python-3.7-pandas-latest
     - test-conda-python-3.7-pandas-master
+    - test-conda-python-3.8-pandas-latest
     - test-conda-python-3.7-dask-latest
-    - test-conda-python-3.7-dask-master
+    - test-conda-python-3.8-dask-master
     - test-conda-python-3.7-turbodbc-latest
     - test-conda-python-3.7-turbodbc-master
     - test-conda-python-3.7-hdfs-2.9.2
@@ -1696,6 +1702,19 @@ tasks:
         - docker-compose build conda-python
         - docker-compose run conda-python
 
+  test-conda-python-3.8:
+    ci: circle
+    platform: linux
+    template: docker-tests/circle.linux.yml
+    params:
+      commands:
+        - export PYTHON=3.8
+        - docker-compose pull --ignore-pull-failures conda-cpp
+        - docker-compose pull --ignore-pull-failures conda-python
+        - docker-compose build conda-cpp
+        - docker-compose build conda-python
+        - docker-compose run conda-python
+
   test-debian-10-python-3:
     ci: circle
     platform: linux
@@ -1864,6 +1883,20 @@ tasks:
         - docker-compose build --no-cache conda-python-pandas
         - docker-compose run conda-python-pandas
 
+  test-conda-python-3.8-pandas-latest:
+    ci: circle
+    platform: linux
+    template: docker-tests/circle.linux.yml
+    params:
+      commands:
+        - export PYTHON=3.8 PANDAS=latest
+        - docker-compose pull --ignore-pull-failures conda-cpp
+        - docker-compose pull --ignore-pull-failures conda-python
+        - docker-compose build conda-cpp
+        - docker-compose build conda-python
+        - docker-compose build --no-cache conda-python-pandas
+        - docker-compose run conda-python-pandas
+
   test-conda-python-3.7-pandas-master:
     ci: circle
     platform: linux
@@ -1892,13 +1925,13 @@ tasks:
         - docker-compose build --no-cache conda-python-dask
         - docker-compose run conda-python-dask
 
-  test-conda-python-3.7-dask-master:
+  test-conda-python-3.8-dask-master:
     ci: circle
     platform: linux
     template: docker-tests/circle.linux.yml
     params:
       commands:
-        - export PYTHON=3.7 DASK=master
+        - export PYTHON=3.8 DASK=master
         - docker-compose pull --ignore-pull-failures conda-cpp
         - docker-compose pull --ignore-pull-failures conda-python
         - docker-compose build conda-cpp

docker-compose.yml

@@ -78,7 +78,8 @@ services:
       - ${ARCH}-conda-cache:/build:delegated
     entrypoint: &conda-entrypoint /bin/bash -c -l
     command: &cpp-conda-command
-      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["conda activate testenv &&
+        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/cpp_test.sh /arrow /build/cpp"]
 
   cuda-cpp:
@@ -289,7 +290,8 @@ services:
     volumes: *conda-volumes
     entrypoint: *conda-entrypoint
     command: &python-conda-command
-      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["conda activate testenv &&
+        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/python_build.sh /arrow /build/python &&
         /arrow/ci/scripts/python_test.sh /arrow"]
 
@@ -454,7 +456,8 @@ services:
     volumes: *conda-volumes
     entrypoint: *conda-entrypoint
     command:
-      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["conda activate testenv &&
+        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/python_build.sh /arrow /build/python &&
         /arrow/ci/scripts/integration_dask.sh"]
 
@@ -483,7 +486,8 @@ services:
     volumes: *conda-volumes
     entrypoint: *conda-entrypoint
     command:
-      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["conda activate testenv &&
+        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/python_build.sh /arrow /build/python &&
         /arrow/ci/scripts/integration_turbodbc.sh /turbodbc /build/turbodbc"]
 
@@ -544,7 +548,8 @@ services:
     volumes: *conda-volumes
     entrypoint: *conda-entrypoint
     command:
-      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["conda activate testenv &&
+        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/r_build.sh /arrow &&
         /arrow/ci/scripts/r_test.sh /arrow"]
 
@@ -759,7 +764,8 @@ services:
       # tell archery where the arrow binaries are located
       ARROW_CPP_EXE_PATH: /build/cpp/debug
     command:
-      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["conda activate testenv &&
+        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/go_build.sh /arrow &&
         /arrow/ci/scripts/java_build.sh /arrow &&
         /arrow/ci/scripts/js_build.sh /arrow &&
@@ -888,7 +894,8 @@ services:
     volumes: *conda-volumes
     entrypoint: *conda-entrypoint
     command:
-      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["conda activate testenv &&
+        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/integration_hiveserver2.sh /arrow /build/cpp"]
 
   conda-python-hdfs:
@@ -925,7 +932,8 @@ services:
       - ${ARCH}-conda-cache:/build:delegated
     entrypoint: *conda-entrypoint
     command:
-      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["conda activate testenv &&
+        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/python_build.sh /arrow /build/python &&
         /arrow/ci/scripts/integration_hdfs.sh /arrow /build/cpp"]
 
@@ -954,7 +962,8 @@ services:
     volumes: *conda-maven-volumes
     entrypoint: *conda-entrypoint
     command:
-      ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
+      ["conda activate testenv &&
+        /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
         /arrow/ci/scripts/python_build.sh /arrow /build/python &&
         /arrow/ci/scripts/java_build.sh /arrow &&
         /arrow/ci/scripts/integration_spark.sh /arrow /spark"]

python/setup.py

@@ -191,15 +191,16 @@ class build_ext(_build_ext):
         source = osp.dirname(osp.abspath(__file__))
 
         # The staging directory for the module being built
-        build_temp = pjoin(os.getcwd(), self.build_temp)
-        build_lib = os.path.join(os.getcwd(), self.build_lib)
+        build_cmd = self.get_finalized_command('build')
+        build_temp = pjoin(os.getcwd(), build_cmd.build_temp)
+        build_lib = pjoin(os.getcwd(), build_cmd.build_lib)
         saved_cwd = os.getcwd()
 
-        if not os.path.isdir(self.build_temp):
-            self.mkpath(self.build_temp)
+        if not os.path.isdir(build_temp):
+            self.mkpath(build_temp)
 
         # Change to the build directory
-        with changed_dir(self.build_temp):
+        with changed_dir(build_temp):
             # Detect if we built elsewhere
             if os.path.isfile('CMakeCache.txt'):
                 cachefile = open('CMakeCache.txt', 'r')