GH-45920: [Release][Python] Upload sdist and wheels to GitHub Releases not apache.jfrog.io (#45962)

### Rationale for this change

We want to stop using apache.jfrog.io. See also: #40760

### What changes are included in this PR?

Use GitHub Release instead of apache.jfrog.io.

Users use only sdist and wheels published to PyPI. Publishing to PyPI isn't changed. So we don't need any migration path for users.

### Are these changes tested?

No. I want to try this in the next release.

### Are there any user-facing changes?

No.
* GitHub Issue: #45920

Authored-by: Sutou Kouhei <kou@clear-code.com>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>

.pre-commit-config.yaml

@@ -195,9 +195,27 @@ repos:
           ?^ci/scripts/c_glib_build\.sh$|
           ?^ci/scripts/c_glib_test\.sh$|
           ?^c_glib/test/run-test\.sh$|
+          ?^dev/release/05-binary-upload\.sh$|
           ?^dev/release/07-binary-verify\.sh$|
+          ?^dev/release/post-03-binary\.sh$|
+          ?^dev/release/post-11-python\.sh$|
           ?^dev/release/utils-generate-checksum\.sh$|
           )
+  - repo: https://github.com/scop/pre-commit-shfmt
+    rev: v3.11.0-1
+    hooks:
+      - id: shfmt
+        args:
+          # The default args is "--write --simplify" but we don't use
+          # "--simplify". Because it's conflicted will ShellCheck.
+          - "--write"
+        # TODO: Remove this when we fix all lint failures
+        files: >-
+          (
+          ?^dev/release/05-binary-upload\.sh$|
+          ?^dev/release/post-03-binary\.sh$|
+          ?^dev/release/post-11-python\.sh$|
+          )
   - repo: https://github.com/trim21/pre-commit-mirror-meson
     rev: v1.6.1
     hooks:

.shellcheckrc

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+external-sources=true
+source-path=SCRIPTDIR

dev/release/.env.example

@@ -42,3 +42,8 @@
 #
 # You must set this.
 #ASF_PASSWORD=secret
+
+# The GitHub token to upload artifacts to GitHub Release.
+#
+# You must set this.
+#GH_TOKEN=secret

dev/release/05-binary-upload.sh

@@ -38,9 +38,10 @@ version_with_rc="${version}-rc${rc}"
 crossbow_job_prefix="release-${version_with_rc}"
 crossbow_package_dir="${SOURCE_DIR}/../../packages"
 
-: ${CROSSBOW_JOB_NUMBER:="0"}
-: ${CROSSBOW_JOB_ID:="${crossbow_job_prefix}-${CROSSBOW_JOB_NUMBER}"}
-: ${ARROW_ARTIFACTS_DIR:="${crossbow_package_dir}/${CROSSBOW_JOB_ID}"}
+: "${CROSSBOW_JOB_NUMBER:=0}"
+: "${CROSSBOW_JOB_ID:=${crossbow_job_prefix}-${CROSSBOW_JOB_NUMBER}}"
+: "${ARROW_ARTIFACTS_DIR:=${crossbow_package_dir}/${CROSSBOW_JOB_ID}}"
+: "${GITHUB_REPOSITORY:=apache/arrow}"
 
 if [ ! -e "${ARROW_ARTIFACTS_DIR}" ]; then
   echo "${ARROW_ARTIFACTS_DIR} does not exist"
@@ -59,6 +60,7 @@ if [ ! -f .env ]; then
   echo "You can use $(pwd)/.env.example as template"
   exit 1
 fi
+# shellcheck source=SCRIPTDIR/.env.example
 . .env
 
 . utils-binary.sh
@@ -66,52 +68,74 @@ fi
 # By default upload all artifacts.
 # To deactivate one category, deactivate the category and all of its dependents.
 # To explicitly select one category, set UPLOAD_DEFAULT=0 UPLOAD_X=1.
-: ${UPLOAD_DEFAULT:=1}
-: ${UPLOAD_ALMALINUX:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_AMAZON_LINUX:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_CENTOS:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_DEBIAN:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_DOCS:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_PYTHON:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_R:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_UBUNTU:=${UPLOAD_DEFAULT}}
+: "${UPLOAD_DEFAULT:=1}"
+: "${UPLOAD_ALMALINUX:=${UPLOAD_DEFAULT}}"
+: "${UPLOAD_AMAZON_LINUX:=${UPLOAD_DEFAULT}}"
+: "${UPLOAD_CENTOS:=${UPLOAD_DEFAULT}}"
+: "${UPLOAD_DEBIAN:=${UPLOAD_DEFAULT}}"
+: "${UPLOAD_DOCS:=${UPLOAD_DEFAULT}}"
+: "${UPLOAD_PYTHON:=${UPLOAD_DEFAULT}}"
+: "${UPLOAD_R:=${UPLOAD_DEFAULT}}"
+: "${UPLOAD_UBUNTU:=${UPLOAD_DEFAULT}}"
+
+tmp_dir=binary/tmp
+rm -rf "${tmp_dir}"
+mkdir -p "${tmp_dir}"
+
+if [ "${UPLOAD_PYTHON}" -gt 0 ]; then
+  dist_dir="${tmp_dir}/dist"
+  mkdir -p "${dist_dir}"
+  for target in "${ARROW_ARTIFACTS_DIR}"/python-sdist/* \
+    "${ARROW_ARTIFACTS_DIR}"/wheel-*/*; do
+    base_name="$(basename "${target}")"
+    cp -a "${target}" "${dist_dir}/${base_name}"
+    gpg \
+      --armor \
+      --detach-sign \
+      --local-user "${GPG_KEY_ID}" \
+      --output "${dist_dir}/${base_name}.asc" \
+      "${target}"
+    pushd "${dist_dir}"
+    shasum -a 512 "${base_name}" >"${base_name}.sha512"
+    popd
+  done
+  gh release upload \
+    --repo apache/arrow \
+    "apache-arrow-${version}-rc${rc}" \
+    "${dist_dir}"/*
+fi
 
 rake_tasks=()
 apt_targets=()
 yum_targets=()
-if [ ${UPLOAD_ALMALINUX} -gt 0 ]; then
+if [ "${UPLOAD_ALMALINUX}" -gt 0 ]; then
   rake_tasks+=(yum:rc:artifactory yum:rc)
   yum_targets+=(almalinux)
 fi
-if [ ${UPLOAD_AMAZON_LINUX} -gt 0 ]; then
+if [ "${UPLOAD_AMAZON_LINUX}" -gt 0 ]; then
   rake_tasks+=(yum:rc:artifactory yum:rc)
   yum_targets+=(amazon-linux)
 fi
-if [ ${UPLOAD_CENTOS} -gt 0 ]; then
+if [ "${UPLOAD_CENTOS}" -gt 0 ]; then
   rake_tasks+=(yum:rc:artifactory yum:rc)
   yum_targets+=(centos)
 fi
-if [ ${UPLOAD_DEBIAN} -gt 0 ]; then
+if [ "${UPLOAD_DEBIAN}" -gt 0 ]; then
   rake_tasks+=(apt:rc:artifactory apt:rc)
   apt_targets+=(debian)
 fi
-if [ ${UPLOAD_DOCS} -gt 0 ]; then
+if [ "${UPLOAD_DOCS}" -gt 0 ]; then
   rake_tasks+=(docs:rc)
 fi
-if [ ${UPLOAD_PYTHON} -gt 0 ]; then
-  rake_tasks+=(python:rc)
-fi
-if [ ${UPLOAD_R} -gt 0 ]; then
+if [ "${UPLOAD_R}" -gt 0 ]; then
   rake_tasks+=(r:rc)
 fi
-if [ ${UPLOAD_UBUNTU} -gt 0 ]; then
+if [ "${UPLOAD_UBUNTU}" -gt 0 ]; then
   rake_tasks+=(apt:rc:artifactory apt:rc)
   apt_targets+=(ubuntu)
 fi
 rake_tasks+=(summary:rc)
 
-tmp_dir=binary/tmp
-mkdir -p "${tmp_dir}"
 source_artifacts_dir="${tmp_dir}/artifacts"
 rm -rf "${source_artifacts_dir}"
 cp -a "${ARROW_ARTIFACTS_DIR}" "${source_artifacts_dir}"
@@ -119,17 +143,23 @@ cp -a "${ARROW_ARTIFACTS_DIR}" "${source_artifacts_dir}"
 docker_run \
   ./runner.sh \
   rake \
-    "${rake_tasks[@]}" \
-    APT_TARGETS=$(IFS=,; echo "${apt_targets[*]}") \
-    ARTIFACTORY_API_KEY="${ARTIFACTORY_API_KEY}" \
-    ARTIFACTS_DIR="${tmp_dir}/artifacts" \
-    ASF_PASSWORD="${ASF_PASSWORD}" \
-    ASF_USER="${ASF_USER}" \
-    DEB_PACKAGE_NAME=${DEB_PACKAGE_NAME:-} \
-    DRY_RUN=${DRY_RUN:-no} \
-    GPG_KEY_ID="${GPG_KEY_ID}" \
-    RC=${rc} \
-    STAGING=${STAGING:-no} \
-    VERBOSE=${VERBOSE:-no} \
-    VERSION=${version} \
-    YUM_TARGETS=$(IFS=,; echo "${yum_targets[*]}")
+  "${rake_tasks[@]}" \
+  APT_TARGETS="$(
+    IFS=,
+    echo "${apt_targets[*]}"
+  )" \
+  ARTIFACTORY_API_KEY="${ARTIFACTORY_API_KEY}" \
+  ARTIFACTS_DIR="${tmp_dir}/artifacts" \
+  ASF_PASSWORD="${ASF_PASSWORD}" \
+  ASF_USER="${ASF_USER}" \
+  DEB_PACKAGE_NAME="${DEB_PACKAGE_NAME:-}" \
+  DRY_RUN="${DRY_RUN:-no}" \
+  GPG_KEY_ID="${GPG_KEY_ID}" \
+  RC="${rc}" \
+  STAGING="${STAGING:-no}" \
+  VERBOSE="${VERBOSE:-no}" \
+  VERSION="${version}" \
+  YUM_TARGETS="$(
+    IFS=,
+    echo "${yum_targets[*]}"
+  )"

dev/release/binary-task.rb

@@ -1124,7 +1124,6 @@ class BinaryTask
     define_apt_tasks
     define_yum_tasks
     define_docs_tasks
-    define_python_tasks
     define_r_tasks
     define_summary_tasks
   end
@@ -2527,14 +2526,6 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}";
                               "test-debian-12-docs/**/*")
   end
 
-  def define_python_tasks
-    define_generic_data_tasks("Python",
-                              :python,
-                              "#{rc_dir}/python/#{full_version}",
-                              "#{release_dir}/python/#{full_version}",
-                              "{python-sdist,wheel-*}/**/*")
-  end
-
   def define_r_rc_tasks(label, id, rc_dir)
     directory rc_dir
 

dev/release/post-03-binary.sh

@@ -37,6 +37,7 @@ if [ ! -f .env ]; then
   echo "You can use $(pwd)/.env.example as template"
   exit 1
 fi
+# shellcheck source=SCRIPTDIR/.env.example
 . .env
 
 . utils-binary.sh
@@ -44,45 +45,41 @@ fi
 # By default deploy all artifacts.
 # To deactivate one category, deactivate the category and all of its dependents.
 # To explicitly select one category, set DEPLOY_DEFAULT=0 DEPLOY_X=1.
-: ${DEPLOY_DEFAULT:=1}
-: ${DEPLOY_ALMALINUX:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_AMAZON_LINUX:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_CENTOS:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_DEBIAN:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_DOCS:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_PYTHON:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_R:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_UBUNTU:=${DEPLOY_DEFAULT}}
+: "${DEPLOY_DEFAULT:=1}"
+: "${DEPLOY_ALMALINUX:=${DEPLOY_DEFAULT}}"
+: "${DEPLOY_AMAZON_LINUX:=${DEPLOY_DEFAULT}}"
+: "${DEPLOY_CENTOS:=${DEPLOY_DEFAULT}}"
+: "${DEPLOY_DEBIAN:=${DEPLOY_DEFAULT}}"
+: "${DEPLOY_DOCS:=${DEPLOY_DEFAULT}}"
+: "${DEPLOY_R:=${DEPLOY_DEFAULT}}"
+: "${DEPLOY_UBUNTU:=${DEPLOY_DEFAULT}}"
 
 rake_tasks=()
 apt_targets=()
 yum_targets=()
-if [ ${DEPLOY_ALMALINUX} -gt 0 ]; then
+if [ "${DEPLOY_ALMALINUX}" -gt 0 ]; then
   rake_tasks+=(yum:artifactory:release)
   yum_targets+=(almalinux)
 fi
-if [ ${DEPLOY_AMAZON_LINUX} -gt 0 ]; then
+if [ "${DEPLOY_AMAZON_LINUX}" -gt 0 ]; then
   rake_tasks+=(yum:artifactory:release)
   yum_targets+=(amazon-linux)
 fi
-if [ ${DEPLOY_CENTOS} -gt 0 ]; then
+if [ "${DEPLOY_CENTOS}" -gt 0 ]; then
   rake_tasks+=(yum:artifactory:release)
   yum_targets+=(centos)
 fi
-if [ ${DEPLOY_DEBIAN} -gt 0 ]; then
+if [ "${DEPLOY_DEBIAN}" -gt 0 ]; then
   rake_tasks+=(apt:artifactory:release)
   apt_targets+=(debian)
 fi
-if [ ${DEPLOY_DOCS} -gt 0 ]; then
+if [ "${DEPLOY_DOCS}" -gt 0 ]; then
   rake_tasks+=(docs:release)
 fi
-if [ ${DEPLOY_PYTHON} -gt 0 ]; then
-  rake_tasks+=(python:release)
-fi
-if [ ${DEPLOY_R} -gt 0 ]; then
+if [ "${DEPLOY_R}" -gt 0 ]; then
   rake_tasks+=(r:release)
 fi
-if [ ${DEPLOY_UBUNTU} -gt 0 ]; then
+if [ "${DEPLOY_UBUNTU}" -gt 0 ]; then
   rake_tasks+=(apt:artifactory:release)
   apt_targets+=(ubuntu)
 fi
@@ -94,13 +91,19 @@ mkdir -p "${tmp_dir}"
 docker_run \
   ./runner.sh \
   rake \
-    --trace \
-    "${rake_tasks[@]}" \
-    APT_TARGETS=$(IFS=,; echo "${apt_targets[*]}") \
-    ARTIFACTORY_API_KEY="${ARTIFACTORY_API_KEY}" \
-    ARTIFACTS_DIR="${tmp_dir}/artifacts" \
-    RC=${rc} \
-    STAGING=${STAGING:-no} \
-    VERBOSE=${VERBOSE:-no} \
-    VERSION=${version} \
-    YUM_TARGETS=$(IFS=,; echo "${yum_targets[*]}")
+  --trace \
+  "${rake_tasks[@]}" \
+  APT_TARGETS="$(
+    IFS=,
+    echo "${apt_targets[*]}"
+  )" \
+  ARTIFACTORY_API_KEY="${ARTIFACTORY_API_KEY}" \
+  ARTIFACTS_DIR="${tmp_dir}/artifacts" \
+  RC="${rc}" \
+  STAGING="${STAGING:-no}" \
+  VERBOSE="${VERBOSE:-no}" \
+  VERSION="${version}" \
+  YUM_TARGETS="$(
+    IFS=,
+    echo "${yum_targets[*]}"
+  )"

dev/release/post-11-python.sh

@@ -21,7 +21,8 @@ set -ex
 set -o pipefail
 
 SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-: ${TEST_PYPI:=0}
+: "${GITHUB_REPOSITORY:=apache/arrow}"
+: "${TEST_PYPI:=0}"
 
 if [ "$#" -ne 1 ]; then
   echo "Usage: $0 <version>"
@@ -30,26 +31,29 @@ fi
 
 version=$1
 
-tmp=$(mktemp -d -t "arrow-post-python.XXXXX")
-base_url=https://apache.jfrog.io/artifactory/arrow/python/${version}
-curl \
-  --location \
-  ${base_url} | \
-  grep -E -o "pyarrow-${version}[a-zA-Z0-9._-]*\\.(tar\\.gz|whl)" | \
-  sort | \
-  uniq | while read artifact; do
-  curl \
-    --fail \
-    --location \
-    --output ${tmp}/${artifact} \
-    ${base_url}/${artifact}
-done
+cd "${SOURCE_DIR}"
 
-if [ ${TEST_PYPI} -gt 0 ]; then
+if [ ! -f .env ]; then
+  echo "You must create $(pwd)/.env"
+  echo "You can use $(pwd)/.env.example as template"
+  exit 1
+fi
+# shellcheck source=SCRIPTDIR/.env.example
+. .env
+
+tmp=$(mktemp -d -t "arrow-post-python.XXXXX")
+gh release download \
+  "apache-arrow-${version}" \
+  --dir "${tmp}" \
+  --pattern "pyarrow-*.tar.gz" \
+  --pattern "pyarrow-*.whl" \
+  --repo "${GITHUB_REPOSITORY}"
+
+if [ "${TEST_PYPI}" -gt 0 ]; then
   TWINE_ARGS="--repository-url https://test.pypi.org/legacy/"
 fi
 
-twine upload ${TWINE_ARGS} ${tmp}/*
+twine upload "${TWINE_ARGS}" "${tmp}"/*
 
 rm -rf "${tmp}"