From 0709cd7e0b866db25f13554b8ded7ef3693c55e3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 30 Mar 2025 10:35:06 +0900 Subject: [PATCH] GH-45920: [Release][Python] Upload sdist and wheels to GitHub Releases not apache.jfrog.io (#45962) ### Rationale for this change We want to stop using apache.jfrog.io. See also: #40760 ### What changes are included in this PR? Use GitHub Release instead of apache.jfrog.io. Users use only sdist and wheels published to PyPI. Publishing to PyPI isn't changed. So we don't need any migration path for users. ### Are these changes tested? No. I want to try this in the next release. ### Are there any user-facing changes? No. * GitHub Issue: #45920 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- .pre-commit-config.yaml | 18 ++++++ .shellcheckrc | 19 ++++++ dev/release/.env.example | 5 ++ dev/release/05-binary-upload.sh | 106 ++++++++++++++++++++------------ dev/release/binary-task.rb | 9 --- dev/release/post-03-binary.sh | 61 +++++++++--------- dev/release/post-11-python.sh | 38 +++++++----- 7 files changed, 163 insertions(+), 93 deletions(-) create mode 100644 .shellcheckrc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 33fc8b87170..1b6cd352874 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -195,9 +195,27 @@ repos: ?^ci/scripts/c_glib_build\.sh$| ?^ci/scripts/c_glib_test\.sh$| ?^c_glib/test/run-test\.sh$| + ?^dev/release/05-binary-upload\.sh$| ?^dev/release/07-binary-verify\.sh$| + ?^dev/release/post-03-binary\.sh$| + ?^dev/release/post-11-python\.sh$| ?^dev/release/utils-generate-checksum\.sh$| ) + - repo: https://github.com/scop/pre-commit-shfmt + rev: v3.11.0-1 + hooks: + - id: shfmt + args: + # The default args is "--write --simplify" but we don't use + # "--simplify". Because it's conflicted will ShellCheck. + - "--write" + # TODO: Remove this when we fix all lint failures + files: >- + ( + ?^dev/release/05-binary-upload\.sh$| + ?^dev/release/post-03-binary\.sh$| + ?^dev/release/post-11-python\.sh$| + ) - repo: https://github.com/trim21/pre-commit-mirror-meson rev: v1.6.1 hooks: diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 00000000000..6eb7b22b947 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +external-sources=true +source-path=SCRIPTDIR diff --git a/dev/release/.env.example b/dev/release/.env.example index 116a39eb567..e751ce4dbc5 100644 --- a/dev/release/.env.example +++ b/dev/release/.env.example @@ -42,3 +42,8 @@ # # You must set this. #ASF_PASSWORD=secret + +# The GitHub token to upload artifacts to GitHub Release. +# +# You must set this. +#GH_TOKEN=secret diff --git a/dev/release/05-binary-upload.sh b/dev/release/05-binary-upload.sh index 1e768c6e356..3c272428629 100755 --- a/dev/release/05-binary-upload.sh +++ b/dev/release/05-binary-upload.sh @@ -38,9 +38,10 @@ version_with_rc="${version}-rc${rc}" crossbow_job_prefix="release-${version_with_rc}" crossbow_package_dir="${SOURCE_DIR}/../../packages" -: ${CROSSBOW_JOB_NUMBER:="0"} -: ${CROSSBOW_JOB_ID:="${crossbow_job_prefix}-${CROSSBOW_JOB_NUMBER}"} -: ${ARROW_ARTIFACTS_DIR:="${crossbow_package_dir}/${CROSSBOW_JOB_ID}"} +: "${CROSSBOW_JOB_NUMBER:=0}" +: "${CROSSBOW_JOB_ID:=${crossbow_job_prefix}-${CROSSBOW_JOB_NUMBER}}" +: "${ARROW_ARTIFACTS_DIR:=${crossbow_package_dir}/${CROSSBOW_JOB_ID}}" +: "${GITHUB_REPOSITORY:=apache/arrow}" if [ ! -e "${ARROW_ARTIFACTS_DIR}" ]; then echo "${ARROW_ARTIFACTS_DIR} does not exist" @@ -59,6 +60,7 @@ if [ ! -f .env ]; then echo "You can use $(pwd)/.env.example as template" exit 1 fi +# shellcheck source=SCRIPTDIR/.env.example . .env . utils-binary.sh @@ -66,52 +68,74 @@ fi # By default upload all artifacts. # To deactivate one category, deactivate the category and all of its dependents. # To explicitly select one category, set UPLOAD_DEFAULT=0 UPLOAD_X=1. -: ${UPLOAD_DEFAULT:=1} -: ${UPLOAD_ALMALINUX:=${UPLOAD_DEFAULT}} -: ${UPLOAD_AMAZON_LINUX:=${UPLOAD_DEFAULT}} -: ${UPLOAD_CENTOS:=${UPLOAD_DEFAULT}} -: ${UPLOAD_DEBIAN:=${UPLOAD_DEFAULT}} -: ${UPLOAD_DOCS:=${UPLOAD_DEFAULT}} -: ${UPLOAD_PYTHON:=${UPLOAD_DEFAULT}} -: ${UPLOAD_R:=${UPLOAD_DEFAULT}} -: ${UPLOAD_UBUNTU:=${UPLOAD_DEFAULT}} +: "${UPLOAD_DEFAULT:=1}" +: "${UPLOAD_ALMALINUX:=${UPLOAD_DEFAULT}}" +: "${UPLOAD_AMAZON_LINUX:=${UPLOAD_DEFAULT}}" +: "${UPLOAD_CENTOS:=${UPLOAD_DEFAULT}}" +: "${UPLOAD_DEBIAN:=${UPLOAD_DEFAULT}}" +: "${UPLOAD_DOCS:=${UPLOAD_DEFAULT}}" +: "${UPLOAD_PYTHON:=${UPLOAD_DEFAULT}}" +: "${UPLOAD_R:=${UPLOAD_DEFAULT}}" +: "${UPLOAD_UBUNTU:=${UPLOAD_DEFAULT}}" + +tmp_dir=binary/tmp +rm -rf "${tmp_dir}" +mkdir -p "${tmp_dir}" + +if [ "${UPLOAD_PYTHON}" -gt 0 ]; then + dist_dir="${tmp_dir}/dist" + mkdir -p "${dist_dir}" + for target in "${ARROW_ARTIFACTS_DIR}"/python-sdist/* \ + "${ARROW_ARTIFACTS_DIR}"/wheel-*/*; do + base_name="$(basename "${target}")" + cp -a "${target}" "${dist_dir}/${base_name}" + gpg \ + --armor \ + --detach-sign \ + --local-user "${GPG_KEY_ID}" \ + --output "${dist_dir}/${base_name}.asc" \ + "${target}" + pushd "${dist_dir}" + shasum -a 512 "${base_name}" >"${base_name}.sha512" + popd + done + gh release upload \ + --repo apache/arrow \ + "apache-arrow-${version}-rc${rc}" \ + "${dist_dir}"/* +fi rake_tasks=() apt_targets=() yum_targets=() -if [ ${UPLOAD_ALMALINUX} -gt 0 ]; then +if [ "${UPLOAD_ALMALINUX}" -gt 0 ]; then rake_tasks+=(yum:rc:artifactory yum:rc) yum_targets+=(almalinux) fi -if [ ${UPLOAD_AMAZON_LINUX} -gt 0 ]; then +if [ "${UPLOAD_AMAZON_LINUX}" -gt 0 ]; then rake_tasks+=(yum:rc:artifactory yum:rc) yum_targets+=(amazon-linux) fi -if [ ${UPLOAD_CENTOS} -gt 0 ]; then +if [ "${UPLOAD_CENTOS}" -gt 0 ]; then rake_tasks+=(yum:rc:artifactory yum:rc) yum_targets+=(centos) fi -if [ ${UPLOAD_DEBIAN} -gt 0 ]; then +if [ "${UPLOAD_DEBIAN}" -gt 0 ]; then rake_tasks+=(apt:rc:artifactory apt:rc) apt_targets+=(debian) fi -if [ ${UPLOAD_DOCS} -gt 0 ]; then +if [ "${UPLOAD_DOCS}" -gt 0 ]; then rake_tasks+=(docs:rc) fi -if [ ${UPLOAD_PYTHON} -gt 0 ]; then - rake_tasks+=(python:rc) -fi -if [ ${UPLOAD_R} -gt 0 ]; then +if [ "${UPLOAD_R}" -gt 0 ]; then rake_tasks+=(r:rc) fi -if [ ${UPLOAD_UBUNTU} -gt 0 ]; then +if [ "${UPLOAD_UBUNTU}" -gt 0 ]; then rake_tasks+=(apt:rc:artifactory apt:rc) apt_targets+=(ubuntu) fi rake_tasks+=(summary:rc) -tmp_dir=binary/tmp -mkdir -p "${tmp_dir}" source_artifacts_dir="${tmp_dir}/artifacts" rm -rf "${source_artifacts_dir}" cp -a "${ARROW_ARTIFACTS_DIR}" "${source_artifacts_dir}" @@ -119,17 +143,23 @@ cp -a "${ARROW_ARTIFACTS_DIR}" "${source_artifacts_dir}" docker_run \ ./runner.sh \ rake \ - "${rake_tasks[@]}" \ - APT_TARGETS=$(IFS=,; echo "${apt_targets[*]}") \ - ARTIFACTORY_API_KEY="${ARTIFACTORY_API_KEY}" \ - ARTIFACTS_DIR="${tmp_dir}/artifacts" \ - ASF_PASSWORD="${ASF_PASSWORD}" \ - ASF_USER="${ASF_USER}" \ - DEB_PACKAGE_NAME=${DEB_PACKAGE_NAME:-} \ - DRY_RUN=${DRY_RUN:-no} \ - GPG_KEY_ID="${GPG_KEY_ID}" \ - RC=${rc} \ - STAGING=${STAGING:-no} \ - VERBOSE=${VERBOSE:-no} \ - VERSION=${version} \ - YUM_TARGETS=$(IFS=,; echo "${yum_targets[*]}") + "${rake_tasks[@]}" \ + APT_TARGETS="$( + IFS=, + echo "${apt_targets[*]}" + )" \ + ARTIFACTORY_API_KEY="${ARTIFACTORY_API_KEY}" \ + ARTIFACTS_DIR="${tmp_dir}/artifacts" \ + ASF_PASSWORD="${ASF_PASSWORD}" \ + ASF_USER="${ASF_USER}" \ + DEB_PACKAGE_NAME="${DEB_PACKAGE_NAME:-}" \ + DRY_RUN="${DRY_RUN:-no}" \ + GPG_KEY_ID="${GPG_KEY_ID}" \ + RC="${rc}" \ + STAGING="${STAGING:-no}" \ + VERBOSE="${VERBOSE:-no}" \ + VERSION="${version}" \ + YUM_TARGETS="$( + IFS=, + echo "${yum_targets[*]}" + )" diff --git a/dev/release/binary-task.rb b/dev/release/binary-task.rb index 8165885ab6b..86056ec9d01 100644 --- a/dev/release/binary-task.rb +++ b/dev/release/binary-task.rb @@ -1124,7 +1124,6 @@ class BinaryTask define_apt_tasks define_yum_tasks define_docs_tasks - define_python_tasks define_r_tasks define_summary_tasks end @@ -2527,14 +2526,6 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; "test-debian-12-docs/**/*") end - def define_python_tasks - define_generic_data_tasks("Python", - :python, - "#{rc_dir}/python/#{full_version}", - "#{release_dir}/python/#{full_version}", - "{python-sdist,wheel-*}/**/*") - end - def define_r_rc_tasks(label, id, rc_dir) directory rc_dir diff --git a/dev/release/post-03-binary.sh b/dev/release/post-03-binary.sh index 493521a1748..4ca545885f4 100755 --- a/dev/release/post-03-binary.sh +++ b/dev/release/post-03-binary.sh @@ -37,6 +37,7 @@ if [ ! -f .env ]; then echo "You can use $(pwd)/.env.example as template" exit 1 fi +# shellcheck source=SCRIPTDIR/.env.example . .env . utils-binary.sh @@ -44,45 +45,41 @@ fi # By default deploy all artifacts. # To deactivate one category, deactivate the category and all of its dependents. # To explicitly select one category, set DEPLOY_DEFAULT=0 DEPLOY_X=1. -: ${DEPLOY_DEFAULT:=1} -: ${DEPLOY_ALMALINUX:=${DEPLOY_DEFAULT}} -: ${DEPLOY_AMAZON_LINUX:=${DEPLOY_DEFAULT}} -: ${DEPLOY_CENTOS:=${DEPLOY_DEFAULT}} -: ${DEPLOY_DEBIAN:=${DEPLOY_DEFAULT}} -: ${DEPLOY_DOCS:=${DEPLOY_DEFAULT}} -: ${DEPLOY_PYTHON:=${DEPLOY_DEFAULT}} -: ${DEPLOY_R:=${DEPLOY_DEFAULT}} -: ${DEPLOY_UBUNTU:=${DEPLOY_DEFAULT}} +: "${DEPLOY_DEFAULT:=1}" +: "${DEPLOY_ALMALINUX:=${DEPLOY_DEFAULT}}" +: "${DEPLOY_AMAZON_LINUX:=${DEPLOY_DEFAULT}}" +: "${DEPLOY_CENTOS:=${DEPLOY_DEFAULT}}" +: "${DEPLOY_DEBIAN:=${DEPLOY_DEFAULT}}" +: "${DEPLOY_DOCS:=${DEPLOY_DEFAULT}}" +: "${DEPLOY_R:=${DEPLOY_DEFAULT}}" +: "${DEPLOY_UBUNTU:=${DEPLOY_DEFAULT}}" rake_tasks=() apt_targets=() yum_targets=() -if [ ${DEPLOY_ALMALINUX} -gt 0 ]; then +if [ "${DEPLOY_ALMALINUX}" -gt 0 ]; then rake_tasks+=(yum:artifactory:release) yum_targets+=(almalinux) fi -if [ ${DEPLOY_AMAZON_LINUX} -gt 0 ]; then +if [ "${DEPLOY_AMAZON_LINUX}" -gt 0 ]; then rake_tasks+=(yum:artifactory:release) yum_targets+=(amazon-linux) fi -if [ ${DEPLOY_CENTOS} -gt 0 ]; then +if [ "${DEPLOY_CENTOS}" -gt 0 ]; then rake_tasks+=(yum:artifactory:release) yum_targets+=(centos) fi -if [ ${DEPLOY_DEBIAN} -gt 0 ]; then +if [ "${DEPLOY_DEBIAN}" -gt 0 ]; then rake_tasks+=(apt:artifactory:release) apt_targets+=(debian) fi -if [ ${DEPLOY_DOCS} -gt 0 ]; then +if [ "${DEPLOY_DOCS}" -gt 0 ]; then rake_tasks+=(docs:release) fi -if [ ${DEPLOY_PYTHON} -gt 0 ]; then - rake_tasks+=(python:release) -fi -if [ ${DEPLOY_R} -gt 0 ]; then +if [ "${DEPLOY_R}" -gt 0 ]; then rake_tasks+=(r:release) fi -if [ ${DEPLOY_UBUNTU} -gt 0 ]; then +if [ "${DEPLOY_UBUNTU}" -gt 0 ]; then rake_tasks+=(apt:artifactory:release) apt_targets+=(ubuntu) fi @@ -94,13 +91,19 @@ mkdir -p "${tmp_dir}" docker_run \ ./runner.sh \ rake \ - --trace \ - "${rake_tasks[@]}" \ - APT_TARGETS=$(IFS=,; echo "${apt_targets[*]}") \ - ARTIFACTORY_API_KEY="${ARTIFACTORY_API_KEY}" \ - ARTIFACTS_DIR="${tmp_dir}/artifacts" \ - RC=${rc} \ - STAGING=${STAGING:-no} \ - VERBOSE=${VERBOSE:-no} \ - VERSION=${version} \ - YUM_TARGETS=$(IFS=,; echo "${yum_targets[*]}") + --trace \ + "${rake_tasks[@]}" \ + APT_TARGETS="$( + IFS=, + echo "${apt_targets[*]}" + )" \ + ARTIFACTORY_API_KEY="${ARTIFACTORY_API_KEY}" \ + ARTIFACTS_DIR="${tmp_dir}/artifacts" \ + RC="${rc}" \ + STAGING="${STAGING:-no}" \ + VERBOSE="${VERBOSE:-no}" \ + VERSION="${version}" \ + YUM_TARGETS="$( + IFS=, + echo "${yum_targets[*]}" + )" diff --git a/dev/release/post-11-python.sh b/dev/release/post-11-python.sh index 65ef44afa66..f94c229a42f 100755 --- a/dev/release/post-11-python.sh +++ b/dev/release/post-11-python.sh @@ -21,7 +21,8 @@ set -ex set -o pipefail SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -: ${TEST_PYPI:=0} +: "${GITHUB_REPOSITORY:=apache/arrow}" +: "${TEST_PYPI:=0}" if [ "$#" -ne 1 ]; then echo "Usage: $0 " @@ -30,26 +31,29 @@ fi version=$1 -tmp=$(mktemp -d -t "arrow-post-python.XXXXX") -base_url=https://apache.jfrog.io/artifactory/arrow/python/${version} -curl \ - --location \ - ${base_url} | \ - grep -E -o "pyarrow-${version}[a-zA-Z0-9._-]*\\.(tar\\.gz|whl)" | \ - sort | \ - uniq | while read artifact; do - curl \ - --fail \ - --location \ - --output ${tmp}/${artifact} \ - ${base_url}/${artifact} -done +cd "${SOURCE_DIR}" -if [ ${TEST_PYPI} -gt 0 ]; then +if [ ! -f .env ]; then + echo "You must create $(pwd)/.env" + echo "You can use $(pwd)/.env.example as template" + exit 1 +fi +# shellcheck source=SCRIPTDIR/.env.example +. .env + +tmp=$(mktemp -d -t "arrow-post-python.XXXXX") +gh release download \ + "apache-arrow-${version}" \ + --dir "${tmp}" \ + --pattern "pyarrow-*.tar.gz" \ + --pattern "pyarrow-*.whl" \ + --repo "${GITHUB_REPOSITORY}" + +if [ "${TEST_PYPI}" -gt 0 ]; then TWINE_ARGS="--repository-url https://test.pypi.org/legacy/" fi -twine upload ${TWINE_ARGS} ${tmp}/* +twine upload "${TWINE_ARGS}" "${tmp}"/* rm -rf "${tmp}"