mirror of
https://github.com/apache/arrow.git
synced 2026-10-28 16:47:19 +08:00
Antoine Pitrou
503cee9aee
### Rationale for this change Our current ccache support for PyArrow currently has two issues: 1. The custom configuration is set using environment variables, but those are not passed to the build processes according to the [CMake doc](https://cmake.org/cmake/help/latest/command/set.html#set-environment-variable) (I have verified this by enabling ccache logging to see the configuration values for each build command) 2. Cython-generated C++ sources are written in the temporary build directory (because of build isolation), and therefore their path changes everytime. Consequently, ccache would fail reusing previous compilation results. ### What changes are included in this PR? 1. Pass custom ccache configuration using command-line arguments, not environment variables. 2. Set the ccache configuration variable `base_dir` to the temporary build directory so that ccache strips away the build directory and hits previously cached results obtained from a different build directory. ### Are these changes tested? Yes, locally I confirmed that ccache now efficiently reuses compilation outputs for Cython-generated C++ sources. Before: ``` real 0m50,313s user 4m22,991s sys 0m24,805s ``` After: ``` real 0m18,698s user 1m14,727s sys 0m7,518s ``` ### Are there any user-facing changes? No. * GitHub Issue: #51138 Authored-by: Antoine Pitrou <antoine@python.org> Signed-off-by: Antoine Pitrou <antoine@python.org>
120 lines
3.9 KiB
Bash
Executable File
120 lines
3.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
set -ex
|
|
|
|
arrow_dir=${1}
|
|
build_dir=${2}
|
|
|
|
source_dir=${arrow_dir}/python
|
|
python_build_dir=${build_dir}/python
|
|
|
|
: "${BUILD_DOCS_PYTHON:=OFF}"
|
|
|
|
if [ -x "$(command -v git)" ]; then
|
|
git config --global --add safe.directory "${arrow_dir}"
|
|
fi
|
|
|
|
if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
|
|
# We don't need to follow this external file.
|
|
# See also: https://www.shellcheck.net/wiki/SC1091
|
|
#
|
|
# shellcheck source=/dev/null
|
|
. "${ARROW_PYTHON_VENV}/bin/activate"
|
|
fi
|
|
|
|
case "$(uname)" in
|
|
Linux)
|
|
n_jobs=$(nproc)
|
|
;;
|
|
Darwin)
|
|
n_jobs=$(sysctl -n hw.ncpu)
|
|
;;
|
|
MINGW*)
|
|
n_jobs=${NUMBER_OF_PROCESSORS:-1}
|
|
;;
|
|
*)
|
|
n_jobs=${NPROC:-1}
|
|
;;
|
|
esac
|
|
|
|
if [ -n "${CONDA_PREFIX}" ]; then
|
|
echo -e "===\n=== Conda environment for build\n==="
|
|
conda list
|
|
fi
|
|
|
|
export CMAKE_BUILD_PARALLEL_LEVEL=${n_jobs}
|
|
export CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
|
|
export PYARROW_WITH_ACERO=${ARROW_ACERO:-OFF}
|
|
export PYARROW_WITH_AZURE=${ARROW_AZURE:-OFF}
|
|
export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF}
|
|
export PYARROW_WITH_DATASET=${ARROW_DATASET:-ON}
|
|
export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF}
|
|
export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA:-OFF}
|
|
export PYARROW_WITH_GCS=${ARROW_GCS:-OFF}
|
|
export PYARROW_WITH_HDFS=${ARROW_HDFS:-ON}
|
|
export PYARROW_WITH_ORC=${ARROW_ORC:-OFF}
|
|
export PYARROW_WITH_PARQUET=${ARROW_PARQUET:-OFF}
|
|
export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION:-ON}
|
|
export PYARROW_WITH_S3=${ARROW_S3:-OFF}
|
|
export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT:-OFF}
|
|
|
|
: "${CMAKE_PREFIX_PATH:=${ARROW_HOME}}"
|
|
export CMAKE_PREFIX_PATH
|
|
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
|
|
export DYLD_LIBRARY_PATH=${ARROW_HOME}/lib${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}}
|
|
|
|
# https://github.com/apache/arrow/issues/41429
|
|
# TODO: We want to out-of-source build. This is a workaround. We copy
|
|
# all needed files to the build directory from the source directory
|
|
# and build in the build directory.
|
|
rm -rf "${python_build_dir}"
|
|
cp -aL "${source_dir}" "${python_build_dir}"
|
|
pushd "${python_build_dir}"
|
|
# - Cannot use build isolation as we want to use specific dependency versions
|
|
# (e.g. Numpy, Pandas) on some CI jobs.
|
|
time ${PYTHON:-python} -m pip install --no-deps --no-build-isolation -vv -C cmake.build-type="${CMAKE_BUILD_TYPE:-Debug}" .
|
|
popd
|
|
|
|
if [ "${BUILD_DOCS_PYTHON}" == "ON" ]; then
|
|
# https://github.com/apache/arrow/issues/41429
|
|
# TODO: We want to out-of-source build. This is a workaround.
|
|
#
|
|
# Copy docs/source because the "autosummary_generate = True"
|
|
# configuration generates files to docs/source/python/generated/.
|
|
rm -rf "${python_build_dir}/docs/source"
|
|
mkdir -p "${python_build_dir}/docs"
|
|
cp -a "${arrow_dir}/docs/source" "${python_build_dir}/docs/"
|
|
rm -rf "${python_build_dir}/format"
|
|
cp -a "${arrow_dir}/format" "${python_build_dir}/"
|
|
rm -rf "${python_build_dir}/cpp/examples"
|
|
mkdir -p "${python_build_dir}/cpp"
|
|
cp -a "${arrow_dir}/cpp/examples" "${python_build_dir}/cpp/"
|
|
rm -rf "${python_build_dir}/ci"
|
|
cp -a "${arrow_dir}/ci/" "${python_build_dir}/"
|
|
export ARROW_CPP_DOXYGEN_XML=${build_dir}/cpp/apidoc/xml
|
|
pushd "${build_dir}"
|
|
sphinx-build \
|
|
-j auto \
|
|
-b html \
|
|
"${python_build_dir}/docs/source" \
|
|
"${build_dir}/docs"
|
|
popd
|
|
fi
|