arrow/.github/workflows/python.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Python

on:
  push:
    branches:
      - '**'
      - '!dependabot/**'
    tags:
      - '**'
    paths:
      - '.dockerignore'
      - '.github/workflows/python.yml'
      - 'ci/**'
      - 'cpp/**'
      - 'compose.yaml'
      - 'docs/source/python/**'
      - 'python/**'
  pull_request:
    paths:
      - '.dockerignore'
      - '.github/workflows/python.yml'
      - 'ci/**'
      - 'cpp/**'
      - 'compose.yaml'
      - 'docs/source/python/**'
      - 'python/**'

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  ARCHERY_DEBUG: 1
  DOCKER_VOLUME_PREFIX: ".docker/"

jobs:

  docker:
    name: ${{ matrix.title }}
    runs-on: ubuntu-latest
    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      max-parallel: 20
      matrix:
        name:
          - conda-python-docs
          - conda-python-3.12-nopandas
          - conda-python-3.11-pandas-2.2.2-numpy-2.0.2
          - conda-python-3.14-pandas-latest
          - conda-python-3.13-no-numpy
        include:
          - name: conda-python-docs
            image: conda-python-docs
            title: AMD64 Conda Python 3.12 Sphinx & Numpydoc
            python: "3.12"
          - name: conda-python-3.12-nopandas
            image: conda-python
            title: AMD64 Conda Python 3.12 Without Pandas
            python: "3.12"
          - name: conda-python-3.11-pandas-2.2.2-numpy-2.0.2
            image: conda-python-pandas
            title: AMD64 Conda Python 3.11 Pandas 2.2.2 NumPy 2.0.2
            python: "3.11"
            pandas: "2.2.2"
            numpy: "2.0.2"
          - name: conda-python-3.14-pandas-latest
            image: conda-python-pandas
            title: AMD64 Conda Python 3.14 Pandas latest
            python: "3.14"
            pandas: latest
          - name: conda-python-3.13-no-numpy
            image: conda-python-no-numpy
            title: AMD64 Conda Python 3.13 without NumPy
            python: "3.13"
    env:
      PYTHON: ${{ matrix.python || 3.11 }}
      UBUNTU: ${{ matrix.ubuntu || 22.04 }}
      PANDAS: ${{ matrix.pandas || 'latest' }}
      NUMPY: ${{ matrix.numpy || 'latest' }}
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v7
        with:
          persist-credentials: false
          fetch-depth: 0
          submodules: recursive
      - name: Restore Docker Volumes
        uses: apache/infrastructure-actions/stash/restore@bc817c8e9ad7d8216aab74223e32ea391e233b2c
        with:
          path: .docker
          key: python-${{ matrix.name }}
      - name: Setup Python
        uses: actions/setup-python@v7
        with:
          python-version: 3.12
      - name: Setup Archery
        run: pip install -e dev/archery[docker]
      - name: Execute Docker Build
        env:
          ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
          ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
        run: |
          source ci/scripts/util_enable_core_dumps.sh
          archery docker run ${{ matrix.image }}
      - name: Save Docker Volumes
        if: ${{ !cancelled() }}
        continue-on-error: true
        uses: apache/infrastructure-actions/stash/save@bc817c8e9ad7d8216aab74223e32ea391e233b2c
        with:
          path: .docker
          key: python-${{ matrix.name }}
          include-hidden-files: true
      - name: Docker Push
        if: >-
          success() &&
          github.event_name == 'push' &&
          github.repository == 'apache/arrow' &&
          github.ref_name == 'main'
        env:
          ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
          ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
        continue-on-error: true
        run: archery docker push ${{ matrix.image }}

  macos:
    name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} Python 3
    runs-on: macos-${{ matrix.macos-version }}
    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      max-parallel: 20
      matrix:
        include:
          - architecture: AMD64
            macos-version: "15-intel"
            large-memory-tests: "OFF"
          - architecture: ARM64
            macos-version: "14"
            large-memory-tests: "ON"
    env:
      ARROW_HOME: /tmp/local
      ARROW_AZURE: ON
      ARROW_DATASET: ON
      ARROW_FLIGHT: ON
      ARROW_GANDIVA: ON
      ARROW_GCS: OFF
      ARROW_HDFS: ON
      ARROW_JEMALLOC: ON
      ARROW_ORC: ON
      ARROW_PARQUET: ON
      PARQUET_REQUIRE_ENCRYPTION: ON
      ARROW_PYTHON: ON
      ARROW_S3: ON
      ARROW_SUBSTRAIT: ON
      ARROW_WITH_ZLIB: ON
      ARROW_WITH_LZ4: ON
      ARROW_WITH_BZ2: ON
      ARROW_WITH_ZSTD: ON
      ARROW_WITH_SNAPPY: ON
      ARROW_WITH_BROTLI: ON
      ARROW_BUILD_TESTS: OFF
      PYARROW_TEST_LARGE_MEMORY: ${{ matrix.large-memory-tests }}
      PYTEST_ARGS: "-n auto --durations=40"
      # Current oldest supported version according to https://endoflife.date/macos
      MACOSX_DEPLOYMENT_TARGET: 12.0
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v7
        with:
          persist-credentials: false
          fetch-depth: 0
          submodules: recursive
      - name: Setup Python
        uses: actions/setup-python@v7
        with:
          python-version: '3.11'
      - name: Install Dependencies
        shell: bash
        run: |
          brew bundle --file=cpp/Brewfile
          python -m pip install \
            -r python/requirements-build.txt \
            -r python/requirements-test.txt
      - name: Install MinIO
        run: |
          $(brew --prefix bash)/bin/bash \
            ci/scripts/install_minio.sh latest /usr/local
      - name: Setup ccache
        shell: bash
        run: ci/scripts/ccache_setup.sh
      - name: ccache info
        id: ccache-info
        shell: bash
        run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
      - name: Restore ccache
        uses: apache/infrastructure-actions/stash/restore@bc817c8e9ad7d8216aab74223e32ea391e233b2c
        with:
          path: ${{ steps.ccache-info.outputs.cache-dir }}
          key: python-ccache-macos-${{ matrix.macos-version }}
      - name: Build
        shell: bash
        run: |
          if [ "${{ matrix.macos-version }}" = "15-intel" ]; then
            # This is a workaround.
            #
            # Homebrew uses /usr/local as prefix. So packages
            # installed by Homebrew also use /usr/local/include. We
            # want to include headers for packages installed by
            # Homebrew as system headers to ignore warnings in them.
            # But "-isystem /usr/local/include" isn't used by CMake
            # because /usr/local/include is marked as the default
            # include path. So we disable -Werror to avoid build error
            # by warnings from packages installed by Homebrew.
            export BUILD_WARNING_LEVEL=PRODUCTION
          fi

          python -m pip install wheel
          ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
          ci/scripts/python_build.sh $(pwd) $(pwd)/build
      - name: Save ccache
        if: ${{ !cancelled() }}
        continue-on-error: true
        uses: apache/infrastructure-actions/stash/save@bc817c8e9ad7d8216aab74223e32ea391e233b2c
        with:
          path: ${{ steps.ccache-info.outputs.cache-dir }}
          key: python-ccache-macos-${{ matrix.macos-version }}
      - name: Test
        shell: bash
        run: ci/scripts/python_test.sh $(pwd) $(pwd)/build

  windows:
    name: AMD64 Windows 2022 Python 3.13
    runs-on: windows-2022
    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
    timeout-minutes: 60
    env:
      CCACHE_DIR: C:/ccache
      PYTHON_CMD: "py -3.13"
    steps:
      - name: Disable Crash Dialogs
        run: |
          reg add `
            "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" `
            /v DontShowUI `
            /t REG_DWORD `
            /d 1 `
            /f
      - name: Checkout Arrow
        uses: actions/checkout@v7
        with:
          persist-credentials: false
          fetch-depth: 0
          submodules: recursive
      - name: Setup Python
        uses: actions/setup-python@v7
        with:
          python-version: 3.13
      - name: Install ccache
        shell: bash
        run: |
          ci/scripts/install_ccache.sh 4.13.6 /usr
      - name: Setup ccache
        shell: bash
        run: |
          ci/scripts/ccache_setup.sh
      - name: Generate path variables
        id: path-info
        shell: bash
        run: |
          echo "usr-windows-dir="$(cygpath --absolute --windows /usr)"" >> $GITHUB_OUTPUT
      - name: Restore ccache
        uses: apache/infrastructure-actions/stash/restore@bc817c8e9ad7d8216aab74223e32ea391e233b2c
        with:
          path: ${{ env.CCACHE_DIR }}
          key: python-ccache-windows
      - name: Build Arrow C++ and PyArrow
        shell: cmd
        env:
          USR_WINDOWS_DIR: ${{ steps.path-info.outputs.usr-windows-dir }}
        run: |
          call "ci\scripts\python_build.bat" %cd% "%USR_WINDOWS_DIR%"
      - name: Save ccache
        if: ${{ !cancelled() }}
        continue-on-error: true
        uses: apache/infrastructure-actions/stash/save@bc817c8e9ad7d8216aab74223e32ea391e233b2c
        with:
          path: ${{ env.CCACHE_DIR }}
          key: python-ccache-windows
      - name: Test PyArrow
        shell: cmd
        run: |
          call "ci\scripts\python_test.bat" %cd%