mirrors/arrow
0
0 Fork 0
mirror of https://github.com/apache/arrow.git synced 2026-10-29 09:05:47 +08:00
Code Activity

Refactor docker-compose file and use it with github actions.

Browse Source
This commit is contained in:
Krisztián Szűcs committed 2019-11-08 17:10:41 +01:00
1 parent 9510b70ecc
commit b9c5478890
165 files changed
+4402 -5390
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.dockerignore
+13 -84
View File
@@ -15,89 +15,18 @@
# specific language governing permissions and limitations
# under the License.
.git
docker_cache
docs/_build
# Turn .dockerignore to .dockerallow by excluding everything and explicitly
# allowing specific files and directories. This enables us to quickly add
# dependency files to the docker contect without scanning the whole directory.
# This setup requires to all of our docker containers have arrow's source
# as a mounted directory.
# IDE
.idea
.vscode
*/.vscode
# exclude everything
**
# c_glib
c_glib/build
c_glib/autom4te.cache
c_glib/m4
c_glib/*/*.o
c_glib/*/*.lo
c_glib/*/*.la
c_glib/*/.deps
c_glib/*/.libs
# cpp
cpp/.idea
cpp/build
cpp/*-build
cpp/*_build
cpp/build-debug
cpp/build-release
cpp/build-test
cpp/Testing
cpp/thirdparty
!cpp/thirdparty/jemalloc
!cpp/thirdparty/versions.txt
!cpp/thirdparty/hadoop/include
# python
python/build
python/dist
python/manylinux1/dist
python/*.egg-info
python/*.egg
python/*.pyc
__pycache__/
*/__pycache__/
*/*/__pycache__/
*/*/*/__pycache__/
*.py[cod]
*/*.py[cod]
*/*/*.py[cod]
*/*/*/*.py[cod]
*.so
*/*.so
*/*/*.so
*/*/*/*.so
*.dylib
*/*.dylib
*/*/*.dylib
*/*/*/*.dylib
# JS
js/.npm
js/node_modules
js/jspm_packages
js/logs
js/*.log
js/.esm-cache
js/npm-debug.log*
js/yarn-debug.log*
js/yarn-error.log*
js/.grunt
js/bower_components
js/.lock-wscript
js/build/Release
js/dist
js/targets
js/test/data/**/*.json
js/test/data/**/*.arrow
# Rust
rust/target
rust/arrow/target
rust/parquet/target
rust/datafusion/target
# Site
site/
# include explicitly
!ci/**
!c_glib/Gemfile
!python/requirements*.txt
!python/manylinux1/**
!python/manylinux2010/**
.env
+40
View File
@@ -0,0 +1,40 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# All of the following environment variables are required to set default values
# for the parameters in docker-compose.yml.
ORG=arrowdev
ARCH=amd64
CUDA=10.0
DEBIAN=10
UBUNTU=18.04
FEDORA=29
PYTHON=3.6
RUST=nightly-2019-09-25
GO=1.12
NODE=11
MAVEN=3.5.4
JDK=8
R=3.6.1
PANDAS=latest
DASK=latest
TURBODBC=latest
HDFS=2.9.2
SPARK=master
DOTNET=2.1
R=3.6
.github/workflows/cpp.yml
+179
View File
@@ -0,0 +1,179 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: C++
on:
push:
# paths:
# - 'cpp/**'
pull_request:
# paths:
# - 'cpp/**'
jobs:
docker:
name: ${{ matrix.name }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
image:
- amd64-conda-latest-cpp
- amd64-debian-10-cpp
# - amd64-ubuntu-16.04-cpp
- amd64-ubuntu-18.04-cpp
- amd64-ubuntu-18.04-cpp-cmake-3.2
- amd64-fedora-29-cpp
include:
- name: AMD64 Conda C++
image: amd64-conda-latest-cpp
service: conda-cpp
- name: AMD64 Debian 10 C++
image: amd64-debian-10-cpp
service: debian-cpp
debian: 10
# TODO(kszucs): earlier GCC has a compile error
# - name: AMD64 Ubuntu 16.04 C++
# image: amd64-ubuntu-16.04-cpp
# service: ubuntu-cpp
# ubuntu: 16.04
- name: AMD64 Ubuntu 18.04 C++
image: amd64-ubuntu-18.04-cpp
service: ubuntu-cpp
ubuntu: 18.04
- name: AMD64 Ubuntu 18.04 C++ CMake 3.2
image: amd64-ubuntu-18.04-cpp-cmake-3.2
service: ubuntu-cpp-cmake32
ubuntu: 18.04
- name: AMD64 Fedora 29 C++
image: amd64-fedora-29-cpp
service: fedora-cpp
fedora: 29
env:
DEBIAN: ${{ matrix.debian || 10 }}
UBUNTU: ${{ matrix.ubuntu || 18.04 }}
FEDORA: ${{ matrix.fedora || 29 }}
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Docker Pull
shell: bash
run: docker-compose pull --ignore-pull-failures ${{ matrix.service }}
- name: Docker Build
shell: bash
run: docker-compose build ${{ matrix.service }}
- name: Docker Run
shell: bash
run: docker-compose run ${{ matrix.service }}
- name: Docker Push
if: github.event_name == 'push' # && github.ref == 'master'
shell: bash
run: |
docker login \
-u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push ${{ matrix.service }}
macos:
name: AMD64 MacOS 10.15 C++
runs-on: macos-latest
strategy:
fail-fast: false
env:
ARROW_HOME: /usr/local
# TODO(kszucs): investigate why does it fail
ARROW_JEMALLOC: OFF
ARROW_ORC: OFF
ARROW_FLIGHT: ON
ARROW_PLASMA: ON
ARROW_GANDIVA: ON
ARROW_PARQUET: ON
ARROW_WITH_ZLIB: ON
ARROW_WITH_LZ4: ON
ARROW_WITH_BZ2: ON
ARROW_WITH_ZSTD: ON
ARROW_WITH_SNAPPY: ON
ARROW_WITH_BROTLI: ON
ARROW_BUILD_TESTS: ON
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Install Dependencies
shell: bash
# TODO(kszucs): consider to use run_brew from travis_install_osx.sh
run: brew bundle --file=cpp/Brewfile
- name: Build
shell: bash
run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/cpp/build
- name: Test
shell: bash
run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/cpp/build
windows:
name: AMD64 ${{ matrix.name }} C++
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- windows-latest
include:
- os: windows-latest
name: Windows 2019
generator: Visual Studio 16 2019
env:
CMAKE_ARGS: '-A x64'
CMAKE_GENERATOR: ${{ matrix.generator }}
CMAKE_INSTALL_LIBDIR: bin
CMAKE_INSTALL_PREFIX: /usr
ARROW_HOME: /usr
ARROW_FLIGHT: OFF
ARROW_PARQUET: OFF
ARROW_WITH_ZLIB: OFF
ARROW_WITH_LZ4: OFF
ARROW_WITH_BZ2: OFF
ARROW_WITH_ZSTD: OFF
ARROW_WITH_SNAPPY: OFF
ARROW_WITH_BROTLI: OFF
ARROW_USE_GLOG: OFF
ARROW_BUILD_TESTS: ON
ARROW_TEST_LINKAGE: static
ARROW_BOOST_USE_SHARED: OFF
ARROW_BUILD_SHARED: OFF
ARROW_USE_STATIC_CRT: ON
ARROW_VERBOSE_THIRDPARTY_BUILD: OFF
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Dependencies
shell: bash
run: choco install dependencywalker
- name: Build
shell: bash
run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/cpp/build
- name: Test
shell: bash
run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/cpp/build
.github/workflows/csharp.yml
+66
View File
@@ -0,0 +1,66 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: C#
on:
push:
# paths:
# - 'csharp/**'
pull_request:
# paths:
# - 'csharp/**'
jobs:
# the docker container fails to run because of the ubuntu host versions, see
# https://github.com/dotnet/core/issues/3509
all-platforms:
name: AMD64 ${{ matrix.name }} C# ${{ matrix.dotnet }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-18.04
- macos-latest
- windows-latest
include:
- os: ubuntu-18.04
name: Ubuntu 18.04
dotnet: 2.2.103
- os: macos-latest
name: MacOS 10.15
dotnet: 2.2.103
- os: windows-latest
name: Windows 2019
dotnet: 2.2.103
steps:
- name: Install csharp
uses: actions/setup-dotnet@v1
with:
dotnet-version: ${{ matrix.dotnet }}
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Build
shell: bash
run: ci/scripts/csharp_build.sh $(pwd)
- name: Test
shell: bash
run: ci/scripts/csharp_test.sh $(pwd)
.github/workflows/dev.yml
+115
View File
@@ -0,0 +1,115 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Dev
on:
push:
pull_request:
jobs:
release:
name: Source Release
runs-on: ubuntu-latest
env:
GIT_AUTHOR_NAME: Github Actions
GIT_AUTHOR_EMAIL: github@actions
GIT_COMMITTER_NAME: Github Actions
GIT_COMMITTER_EMAIL: github@actions
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Install Python
uses: actions/setup-python@v1
with:
python-version: '3.6'
- name: Install Ruby
uses: actions/setup-ruby@v1
with:
ruby-version: '2.6'
- name: Install Dependencies
shell: bash
run: |
pip install cython setuptools
wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb
sudo dpkg -i packages-microsoft-prod.deb
sudo apt-get install apt-transport-https
sudo apt-get update
sudo apt-get install dotnet-sdk-2.2
- name: Run Release Test
shell: bash
run: |
ci/scripts/release_test.sh $(pwd)
docker-lint:
name: Lint C++, Python, R, Rust, Docker, RAT
runs-on: ubuntu-latest
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Docker Pull
run: |
docker-compose pull --ignore-pull-failures ubuntu-cpp
docker-compose pull --ignore-pull-failures ubuntu-lint
- name: Docker Build
run: |
docker-compose build ubuntu-cpp
docker-compose build ubuntu-lint
- name: Docker Run
run: docker-compose run ubuntu-lint
- name: Docker Push
if: github.event_name == 'push' # && github.ref == 'master'
shell: bash
run: |
docker login \
-u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push ubuntu-lint
docker-docs:
name: Sphinx and API documentations
runs-on: ubuntu-latest
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Docker Pull
run: |
docker-compose pull --ignore-pull-failures ubuntu-cpp
docker-compose pull --ignore-pull-failures ubuntu-python
docker-compose pull --ignore-pull-failures ubuntu-docs
- name: Docker Build
run: |
docker-compose build ubuntu-cpp
docker-compose build ubuntu-python
docker-compose build ubuntu-docs
- name: Docker Run
run: docker-compose run ubuntu-docs
- name: Docker Push
if: github.event_name == 'push' # && github.ref == 'master'
shell: bash
run: |
docker login \
-u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push ubuntu-docs
.github/workflows/go.yml
+88
View File
@@ -0,0 +1,88 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Go
on:
push:
# paths:
# - 'go/**'
pull_request:
# paths:
# - 'go/**'
jobs:
docker:
name: AMD64 Debian Go ${{ matrix.go }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
go: [1.12]
env:
GO: ${{ matrix.go }}
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Docker Pull
run: docker-compose pull --ignore-pull-failures debian-go
- name: Docker Build
run: docker-compose build debian-go
- name: Docker Run
run: docker-compose run debian-go
- name: Docker Push
if: github.event_name == 'push' # && github.ref == 'master'
shell: bash
run: |
docker login \
-u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push debian-go
macos-windows:
name: AMD64 ${{ matrix.name }} Go ${{ matrix.go }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- macos-latest
- windows-latest
include:
- os: macos-latest
name: MacOS 10.15
go: 1.12
- os: windows-latest
name: Windows 2019
go: 1.12
steps:
- name: Install go
uses: actions/setup-go@v1
with:
go-version: ${{ matrix.go }}
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Build
shell: bash
run: ci/scripts/go_build.sh .
- name: Test
shell: bash
run: ci/scripts/go_test.sh .
.github/workflows/integration.yml
+53
View File
@@ -0,0 +1,53 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Integration
on:
push:
pull_request:
jobs:
docker-integration:
name: AMD64 Conda Integration Test
runs-on: ubuntu-latest
env:
MAVEN: 3.5
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Docker Pull
run: |
docker-compose pull --ignore-pull-failures conda-cpp
docker-compose pull --ignore-pull-failures conda-integration
- name: Docker Build
run: |
docker-compose build conda-cpp
docker-compose build conda-integration
- name: Docker Run
run: docker-compose run conda-integration
- name: Docker Push
if: github.event_name == 'push' # && github.ref == 'master'
shell: bash
run: |
docker login \
-u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push conda-integration
.github/workflows/java.yml
+122
View File
@@ -0,0 +1,122 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Java
on:
push:
# paths:
# - 'cpp/**'
# - 'java/**'
pull_request:
# paths:
# - 'cpp/**'
# - 'java/**'
jobs:
docker:
name: AMD64 Debian 9 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
image:
- amd64-debian-9-java-8-maven-3.5.2
- amd64-debian-9-java-11-maven-3.6.2
include:
- image: amd64-debian-9-java-8-maven-3.5.2
jdk: 8
maven: 3.5.2
- image: amd64-debian-9-java-11-maven-3.6.2
jdk: 11
maven: 3.6.2
env:
JDK: ${{ matrix.jdk }}
MAVEN: ${{ matrix.maven }}
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Docker Pull
run: docker-compose pull --ignore-pull-failures debian-java
- name: Docker Build
run: docker-compose build debian-java
- name: Docker Run
run: docker-compose run debian-java
- name: Docker Push
if: github.event_name == 'push' # && github.ref == 'master'
shell: bash
run: |
docker login \
-u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push debian-java
docker-jni:
name: AMD64 Debian 9 Java JNI and Plasma
runs-on: ubuntu-latest
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Docker Pull
run: |
docker-compose pull --ignore-pull-failures debian-java
docker-compose pull --ignore-pull-failures debian-java-jni
- name: Docker Build
run: |
docker-compose build debian-java
docker-compose build debian-java-jni
- name: Docker Run
run: |
docker-compose run debian-java-jni
- name: Docker Push
if: github.event_name == 'push' # && github.ref == 'master'
shell: bash
run: |
docker login \
-u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push debian-java-jni
macos:
name: AMD64 MacOS 10.15 Java JDK ${{ matrix.jdk }}
runs-on: macos-latest
strategy:
fail-fast: false
matrix:
jdk: [11]
env:
ARROW_JAVA_RUN_TESTS: 1
steps:
- name: Set up Java
uses: actions/setup-java@v1
with:
java-version: ${{ matrix.jdk }}
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Build
shell: bash
run: ci/scripts/java_build.sh $(pwd)
- name: Test
shell: bash
run: ci/scripts/java_test.sh $(pwd)
.github/workflows/js.yml
+118
View File
@@ -0,0 +1,118 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: NodeJS
on:
push:
# paths:
# - 'js/**'
pull_request:
# paths:
# - 'js/**'
jobs:
docker:
name: AMD64 Debian NodeJS ${{ matrix.node }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
node: [11]
env:
NODE: ${{ matrix.node }}
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Docker Pull
run: docker-compose pull --ignore-pull-failures debian-js
- name: Docker Build
run: docker-compose build debian-js
- name: Docker Run
run: docker-compose run debian-js
- name: Docker Push
if: github.event_name == 'push' # && github.ref == 'master'
shell: bash
run: |
docker login \
-u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push debian-js
macos:
name: AMD64 MacOS 10.15 NodeJS ${{ matrix.node }}
runs-on: macos-latest
strategy:
fail-fast: false
matrix:
node: [11]
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Install NodeJS
uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node }}
- name: Build
shell: bash
run: ci/scripts/js_build.sh $(pwd)
- name: Test
shell: bash
run: ci/scripts/js_test.sh $(pwd)
# TODO(kszucs): windows build fails with:
#
# > NODE_NO_WARNINGS=1 gulp build
# 'NODE_NO_WARNINGS' is not recognized as an internal or external command,
# operable program or batch file.
# npm ERR! code ELIFECYCLE
# npm ERR! errno 1
# npm ERR! apache-arrow@1.0.0-SNAPSHOT build: `NODE_NO_WARNINGS=1 gulp build`
# npm ERR! Exit status 1
# npm ERR!
# npm ERR! Failed at the apache-arrow@1.0.0-SNAPSHOT build script.
# npm ERR! This is probably not a problem with npm. There is likely additional logging output above.
#
# windows:
# name: AMD64 Windows 2019 NodeJS ${{ matrix.node }}
# runs-on: windows-latest
# strategy:
# fail-fast: false
# matrix:
# node: [11]
# steps:
# - name: Checkout Arrow
# uses: actions/checkout@v1
# with:
# submodules: true
# - name: Install NodeJS
# uses: actions/setup-node@v1
# with:
# node-version: ${{ matrix.node }}
# - name: Install Platform Dependencies
# shell: bash
# run: npm install -g cross-env
# - name: Build
# shell: bash
# run: ci/scripts/js_build.sh $(pwd)
# - name: Test
# shell: bash
# run: ci/scripts/js_test.sh $(pwd)
.github/workflows/linux-docker-compose.yml
-58
View File
@@ -1,58 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Linux docker-compose
on:
- push
- pull_request
jobs:
build:
name: Test
strategy:
matrix:
label:
- C++ w/ clang-7 & system packages
- Rust
- Lint, Release tests
include:
- label: C++ w/ clang-7 & system packages
image: cpp-system-deps
skip_expression: |
${ARROW_CI_CPP_AFFECTED} != "1"
- label: Rust
image: rust
skip_expression: |
${ARROW_CI_RUST_AFFECTED} != "1"
- label: Lint, Release tests
image: lint
# Don't skip
skip_expression: |
"1" != "1"
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@master
with:
submodules: true
- name: docker-compose
run: |
eval "$(python ci/detect-changes.py)"
if [[ ${{ matrix.skip_expression }} ]]; then
exit
fi
docker-compose build --pull ${{ matrix.image }}
docker-compose run ${{ matrix.image }}
.github/workflows/pull_request.yml
+2
View File
@@ -16,10 +16,12 @@
# under the License.
name: Pull request
on:
schedule:
- cron: |
*/15 * * * *
jobs:
jira-link:
name: JIRA link
.github/workflows/python.yml
+229
View File
@@ -0,0 +1,229 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Python
on:
push:
# paths:
# - 'cpp/**'
# - 'python/**'
pull_request:
# paths:
# - 'cpp/**'
# - 'python/**'
jobs:
docker:
name: ${{ matrix.name }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
image:
- amd64-conda-latest-python-2.7
- amd64-conda-latest-python-3.6
- amd64-conda-latest-python-3.7
- amd64-debian-10-python-3
# - amd64-ubuntu-16.04-python-3
- amd64-ubuntu-18.04-python-3
- amd64-fedora-29-python-3
include:
- name: AMD64 Conda Python 2.7
image: amd64-conda-latest-python-2.7
platform: conda
python: 2.7
- name: AMD64 Conda Python 3.6
image: amd64-conda-latest-python-3.6
platform: conda
python: 3.6
- name: AMD64 Conda Python 3.7
image: amd64-conda-latest-python-3.7
platform: conda
python: 3.7
- name: AMD64 Debian 10 Python 3
image: amd64-debian-10-python-3
platform: debian
debian: 10
python: 3
# - name: AMD64 Ubuntu 16.04 Python 3
# image: amd64-ubuntu-16.04-python-3
# platform: ubuntu
# ubuntu: 16.04
# python: 3
- name: AMD64 Ubuntu 18.04 Python 3
image: amd64-ubuntu-18.04-python-3
platform: ubuntu
ubuntu: 18.04
python: 3
- name: AMD64 Fedora 29 Python 3
image: amd64-fedora-29-python-3
platform: fedora
fedora: 29
python: 3
env:
PYTHON: ${{ matrix.python || 3.7 }}
DEBIAN: ${{ matrix.debian || 10 }}
UBUNTU: ${{ matrix.ubuntu || 18.04 }}
FEDORA: ${{ matrix.fedora || 29 }}
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Docker Pull
shell: bash
run: |
docker-compose pull --ignore-pull-failures ${{ matrix.platform }}-cpp
docker-compose pull --ignore-pull-failures ${{ matrix.platform }}-python
- name: Docker Build
shell: bash
run: |
docker-compose build ${{ matrix.platform }}-cpp
docker-compose build ${{ matrix.platform }}-python
- name: Docker Run
shell: bash
run: docker-compose run ${{ matrix.platform }}-python
- name: Docker Push
if: github.event_name == 'push' # && github.ref == 'master'
shell: bash
run: |
docker login \
-u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push ${{ matrix.platform }}-python
docker-integration:
# TODO(kszucs): we should probably have a test with pandas='master' running
# with a daily cron schedule
name: ${{ matrix.name }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
image:
- amd64-conda-latest-python-3.7-pandas-latest
- amd64-conda-latest-python-3.7-dask-latest
- amd64-conda-latest-python-3.7-turbodbc-latest
include:
- name: AMD64 Conda Python 3.7 Pandas
image: amd64-conda-latest-python-3.7-pandas-latest
platform: conda
integration: pandas
python: 3.7
pandas: latest
- name: AMD64 Conda Python 3.7 Dask
image: amd64-conda-latest-python-3.7-dask-latest
platform: conda
integration: dask
python: 3.7
dask: latest
- name: AMD64 Conda Python 3.7 Turbodbc
image: amd64-conda-latest-python-3.7-turbodbc-latest
platform: conda
integration: turbodbc
python: 3.7
turbodbc: latest
env:
PYTHON: ${{ matrix.python || 3.7 }}
PANDAS: ${{ matrix.pandas || 'latest' }}
DASK: ${{ matrix.dask || 'latest' }}
TURBODBC: ${{ matrix.turbodbc || 'latest' }}
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Docker Pull
shell: bash
run: |
docker-compose pull --ignore-pull-failures \
${{ matrix.platform }}-cpp
docker-compose pull --ignore-pull-failures \
${{ matrix.platform }}-python
docker-compose pull --ignore-pull-failures \
${{ matrix.platform }}-python-${{ matrix.integration }}
- name: Docker Build
shell: bash
run: |
docker-compose build \
${{ matrix.platform }}-cpp
docker-compose build \
${{ matrix.platform }}-python
docker-compose build \
${{ matrix.platform }}-python-${{ matrix.integration }}
- name: Docker Run
shell: bash
run: |
docker-compose run \
${{ matrix.platform }}-python-${{ matrix.integration }}
- name: Docker Push
if: github.event_name == 'push' # && github.ref == 'master'
shell: bash
run: |
docker login \
-u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push \
${{ matrix.platform }}-python-${{ matrix.integration }}
macos:
name: AMD64 MacOS 10.15 Python ${{ matrix.python }}
runs-on: macos-latest
strategy:
fail-fast: false
matrix:
python: [3]
env:
ARROW_HOME: /usr/local
# TODO(kszucs): investigate why does it fail
ARROW_JEMALLOC: OFF
ARROW_ORC: OFF
ARROW_BUILD_TESTS: OFF
ARROW_PYTHON: ON
ARROW_PLASMA: ON
ARROW_GANDIVA: ON
ARROW_PARQUET: ON
ARROW_WITH_ZLIB: ON
ARROW_WITH_LZ4: ON
ARROW_WITH_BZ2: ON
ARROW_WITH_ZSTD: ON
ARROW_WITH_SNAPPY: ON
ARROW_WITH_BROTLI: ON
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Install Dependencies
shell: bash
run: |
brew bundle --file=cpp/Brewfile
brew install coreutils python
pip install -r python/requirements.txt \
-r python/requirements-test.txt \
cython
- name: Build
shell: bash
run: |
ci/scripts/cpp_build.sh $(pwd)
ci/scripts/python_build.sh $(pwd)
- name: Test
shell: bash
run: |
ci/scripts/python_test.sh $(pwd)
.github/workflows/r.yml
+79
View File
@@ -0,0 +1,79 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: R
on:
push:
# paths:
# - 'cpp/**'
# - 'r/**'
pull_request:
# paths:
# - 'cpp/**'
# - 'r/**'
jobs:
docker:
name: ${{ matrix.name }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
image:
- amd64-conda-latest-r-3.6
- amd64-ubuntu-18.04-r-3.6
include:
- name: AMD64 Conda R 3.6
image: amd64-conda-latest-r-3.6
platform: conda
r: 3.6
- name: AMD64 Ubuntu 18.04 R 3.6
image: amd64-ubuntu-18.04-r-3.6
platform: ubuntu
ubuntu: 18.04
r: 3.6
env:
R: ${{ matrix.r || 3.6 }}
UBUNTU: ${{ matrix.ubuntu || 18.04 }}
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Docker Pull
shell: bash
run: |
docker-compose pull --ignore-pull-failures ${{ matrix.platform }}-cpp
docker-compose pull --ignore-pull-failures ${{ matrix.platform }}-r
- name: Docker Build
shell: bash
run: |
docker-compose build ${{ matrix.platform }}-cpp
docker-compose build ${{ matrix.platform }}-r
- name: Docker Run
shell: bash
run: docker-compose run ${{ matrix.platform }}-r
- name: Docker Push
if: github.event_name == 'push' # && github.ref == 'master'
shell: bash
run: |
docker login \
-u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push ${{ matrix.platform }}-r
.github/workflows/ruby.yml
+116
View File
@@ -0,0 +1,116 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: C Glib & Ruby
on:
push:
# paths:
# - 'cpp/**'
# - 'c_glib/**'
# - 'ruby/**'
pull_request:
# paths:
# - 'cpp/**'
# - 'c_glib/**'
# - 'ruby/**'
jobs:
docker:
name: ${{ matrix.name }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
image:
- amd64-ubuntu-18.04-ruby
include:
- name: AMD64 Ubuntu 18.04 Glib & Ruby
image: amd64-ubuntu-18.04-ruby
ubuntu: 18.04
platform: ubuntu
env:
UBUNTU: ${{ matrix.ubuntu || 18.04 }}
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Docker Pull
shell: bash
run: |
docker-compose pull --ignore-pull-failures \
${{ matrix.platform }}-cpp
docker-compose pull --ignore-pull-failures \
${{ matrix.platform }}-c-glib
docker-compose pull --ignore-pull-failures \
${{ matrix.platform }}-ruby
- name: Docker Build
shell: bash
run: |
docker-compose build ${{ matrix.platform }}-cpp
docker-compose build ${{ matrix.platform }}-c-glib
docker-compose build ${{ matrix.platform }}-ruby
- name: Docker Run
shell: bash
run: docker-compose run ${{ matrix.platform }}-ruby
- name: Docker Push
if: github.event_name == 'push' # && github.ref == 'master'
shell: bash
run: |
docker login \
-u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push ${{ matrix.platform }}-ruby
macos:
name: AMD64 MacOS 10.15 Glib & Ruby
runs-on: macos-latest
strategy:
fail-fast: false
env:
ARROW_HOME: /usr/local
ARROW_ORC: OFF
ARROW_JEMALLOC: OFF
ARROW_WITH_BROTLI: ON
ARROW_WITH_LZ4: ON
ARROW_WITH_SNAPPY: ON
ARROW_WITH_ZLIB: ON
ARROW_BUILD_TESTS: ON
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Install Dependencies
shell: bash
run: |
brew bundle --file=cpp/Brewfile
brew bundle --file=c_glib/Brewfile
bundle install --gemfile ruby/red-arrow/Gemfile
- name: Build
shell: bash
run: |
ci/scripts/cpp_build.sh $(pwd) $(pwd)/cpp/build
ci/scripts/c_glib_build.sh $(pwd) $(pwd)/c_glib/build
- name: Test Glib
shell: bash
run: ci/scripts/c_glib_test.sh $(pwd) $(pwd)/c_glib/build
- name: Test Ruby
shell: bash
run: ci/scripts/ruby_test.sh $(pwd)
.github/workflows/rust.yml
+110
View File
@@ -0,0 +1,110 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Rust
on:
push:
# paths:
# - 'rust/**'
pull_request:
# paths:
# - 'rust/**'
jobs:
docker:
name: AMD64 Debian Rust ${{ matrix.rust }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
rust: [nightly-2019-09-25]
env:
RUST: ${{ matrix.rust }}
steps:
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Docker Pull
run: docker-compose pull --ignore-pull-failures debian-rust
- name: Docker Build
run: docker-compose build debian-rust
- name: Docker Run
run: docker-compose run debian-rust
- name: Docker Push
if: github.event_name == 'push' # && github.ref == 'master'
shell: bash
run: |
docker login \
-u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push debian-rust
macos:
name: AMD64 MacOS 10.15 Rust ${{ matrix.rust }}
runs-on: macos-latest
strategy:
fail-fast: false
matrix:
rust: [nightly-2019-09-25]
steps:
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: ${{ matrix.rust }}
override: true
- name: Install Flatbuffers
shell: bash
run: brew install flatbuffers
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Build
shell: bash
run: ci/scripts/rust_build.sh $(pwd) $(pwd)/rust/target
- name: Test
shell: bash
run: ci/scripts/rust_test.sh $(pwd)
windows:
name: AMD64 Windows 2019 Rust ${{ matrix.rust }}
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
rust: [nightly-2019-09-25]
steps:
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: ${{ matrix.rust }}
override: true
- name: Install Flatbuffers
shell: bash
run: choco install flatc
- name: Checkout Arrow
uses: actions/checkout@v1
with:
submodules: true
- name: Build
shell: bash
run: ci/scripts/rust_build.sh $(pwd) $(pwd)/rust/target
- name: Test
shell: bash
run: ci/scripts/rust_test.sh $(pwd)
.github/workflows/windows-msvc-cpp.yml
-54
View File
@@ -1,54 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Windows MSVC C++
on:
- push
- pull_request
jobs:
build:
name: Build
strategy:
matrix:
cmake-generator:
- Visual Studio 16 2019
- Visual Studio 15 2017
include:
- cmake-generator: Visual Studio 16 2019
os: windows-2019
- cmake-generator: Visual Studio 15 2017
os: windows-2016
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@master
- name: CMake
shell: cmd
run: |
mkdir build\cpp
cmake ^
-G "${{ matrix.cmake-generator }}" ^
-A x64 ^
-DARROW_USE_GLOG=OFF ^
-DARROW_WITH_LZ4=OFF ^
-DARROW_WITH_ZLIB=OFF ^
-DCMAKE_INSTALL_PREFIX=install ^
-S cpp ^
-B build\cpp
- name: Install
shell: cmd
run: |
cmake --build build\cpp --config Debug --target Install
.travis.yml
-326
View File
@@ -1,326 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
sudo: required
dist: xenial
services:
- docker
cache:
directories:
- $HOME/.ccache
- $HOME/.m2 # Maven
git:
# ARROW-3083: We use a higher clone depth to also have the latest tag in the cloned history.
depth: 250
before_install:
- source ci/travis/before-install.sh
matrix:
fast_finish: true
include:
# -------------------------------------------------------------------------
- name: "Lint, Release tests"
os: linux
language: python
python: "3.6"
before_script:
# Always run RAT checks, in case another build in matrix breaks RAT
- $TRAVIS_BUILD_DIR/ci/travis_release_audit.sh
- $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
script:
- $TRAVIS_BUILD_DIR/ci/travis_lint.sh
- $TRAVIS_BUILD_DIR/ci/travis_release_test.sh
# -------------------------------------------------------------------------
- name: "C++ unit tests w/ conda-forge toolchain, coverage"
compiler: gcc
language: cpp
os: linux
jdk: openjdk8
env:
- ARROW_BUILD_WARNING_LEVEL=CHECKIN
- ARROW_TRAVIS_COVERAGE=0
- ARROW_TRAVIS_FLIGHT=1
- ARROW_TRAVIS_GANDIVA=1
- ARROW_TRAVIS_GANDIVA_JAVA=1
- ARROW_TRAVIS_ORC=1
- ARROW_TRAVIS_PARQUET=1
- ARROW_TRAVIS_PARQUET_ENCRYPTION=1
- ARROW_TRAVIS_PLASMA=1
- ARROW_TRAVIS_S3=1
- ARROW_TRAVIS_USE_SYSTEM_JAVA=1
- ARROW_TRAVIS_USE_TOOLCHAIN=1
# TODO: This fails in re2 code
# - ARROW_TRAVIS_VALGRIND=1
- ARROW_TRAVIS_VERBOSE=1
# ARROW-6509: disable MIMALLOC on xenial
# - ARROW_TRAVIS_MIMALLOC=1
before_script:
- if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
- $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
- $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh
# If either C++ or Python changed, we must install the C++ libraries
- git submodule update --init
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
- $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1
#- $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1
# -------------------------------------------------------------------------
- name: "Python 3.6 unit tests, conda-forge toolchain, coverage"
compiler: gcc
language: cpp
os: linux
jdk: openjdk8
env:
- ARROW_BUILD_WARNING_LEVEL=CHECKIN
- ARROW_TRAVIS_COVERAGE=0
- ARROW_TRAVIS_FLIGHT=1
- ARROW_TRAVIS_OPTIONAL_INSTALL=1
- ARROW_TRAVIS_PYTHON_DOCS=1
- ARROW_TRAVIS_PYTHON_JVM=1
- ARROW_TRAVIS_USE_SYSTEM_JAVA=1
- ARROW_TRAVIS_USE_TOOLCHAIN=1
- ARROW_TRAVIS_S3=1
# TODO(wesm): Run the benchmarks outside of Travis
# - ARROW_TRAVIS_PYTHON_BENCHMARKS=1
before_script:
- if [ $ARROW_CI_PYTHON_AFFECTED != "1" ] && [ $ARROW_CI_DOCS_AFFECTED != "1" ]; then exit; fi
- $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
- $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh
- $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_java.sh || travis_terminate 1
- export ARROW_TRAVIS_PYTHON_GANDIVA=1
- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
#- $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh
# -------------------------------------------------------------------------
- name: "[OS X] C++ w/ XCode 9.3"
compiler: clang
osx_image: xcode9.3
os: osx
addons:
env:
- ARROW_TRAVIS_USE_TOOLCHAIN=1
- ARROW_TRAVIS_PLASMA=1
- ARROW_TRAVIS_FLIGHT=1
- ARROW_TRAVIS_ORC=1
- ARROW_TRAVIS_S3=1
- ARROW_TRAVIS_PARQUET=1
- ARROW_TRAVIS_PARQUET_ENCRYPTION=1
# TODO(ARROW-4763): llvm and llvmdev packages are in conflict:
# https://github.com/conda-forge/llvmdev-feedstock/issues/60
# - ARROW_TRAVIS_GANDIVA=1
# - ARROW_TRAVIS_GANDIVA_JAVA=1
- ARROW_TRAVIS_OPTIONAL_INSTALL=1
- ARROW_TRAVIS_VERBOSE=1
- ARROW_BUILD_WARNING_LEVEL=CHECKIN
- ARROW_TRAVIS_MIMALLOC=1
before_script:
- if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
# If either C++ or Python changed, we must install the C++ libraries
- git submodule update --init
- $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
# Disabled because of ARROW-4763
# - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh
# -------------------------------------------------------------------------
- name: "[OS X] Python w/ XCode 9.3"
compiler: clang
osx_image: xcode9.3
os: osx
addons:
env:
- ARROW_TRAVIS_S3=1
- ARROW_TRAVIS_PLASMA=1
- ARROW_TRAVIS_USE_TOOLCHAIN=1
- ARROW_BUILD_WARNING_LEVEL=CHECKIN
- ARROW_TRAVIS_OPTIONAL_INSTALL=1
- ARROW_TRAVIS_VERBOSE=1
- MACOSX_DEPLOYMENT_TARGET="10.9"
before_script:
script:
- if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
- $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh
- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
# -------------------------------------------------------------------------
- name: "Java OpenJDK8 and OpenJDK11"
language: cpp
os: linux
before_script:
- if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
- docker-compose build --pull java-all-jdks
script:
- docker-compose run java-all-jdks
# -------------------------------------------------------------------------
- name: "Protocol / Flight Integration Tests"
language: cpp
os: linux
env: ARROW_TEST_GROUP=integration
before_script:
- if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi
- docker-compose pull integration
script:
- docker-compose run integration
# -------------------------------------------------------------------------
- name: "NodeJS"
language: node_js
os: linux
node_js:
- '11.12'
before_script:
- if [ $ARROW_CI_JS_AFFECTED != "1" ]; then exit; fi
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
- $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
# -------------------------------------------------------------------------
- name: "C++ & GLib & Ruby w/ gcc 5.4"
compiler: gcc
os: linux
env:
- ARROW_TRAVIS_ORC=1
- ARROW_TRAVIS_GANDIVA=1
- ARROW_TRAVIS_PARQUET=1
- ARROW_TRAVIS_PLASMA=1
- ARROW_TRAVIS_MIMALLOC=1
before_script:
- if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi
- $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
- $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
- $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh
script:
- if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
- $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh
# -------------------------------------------------------------------------
- name: "[OS X] C++ & GLib & Ruby w/ XCode 10.2 & Homebrew"
compiler: clang
osx_image: xcode10.2
os: osx
env:
- ARROW_TRAVIS_FLIGHT=1
- ARROW_TRAVIS_GANDIVA=1
# Disabled until either ORC-480 or ORC-561 is fixed
- ARROW_TRAVIS_ORC=0
- ARROW_TRAVIS_PARQUET=1
- ARROW_TRAVIS_PLASMA=1
- ARROW_TRAVIS_PYTHON=1
- ARROW_TRAVIS_VERBOSE=1
- ARROW_TRAVIS_MIMALLOC=1
addons:
before_script:
- if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi
- $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library --homebrew
- $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
- $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh
script:
- if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
- $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh
# -------------------------------------------------------------------------
- name: Go
language: go
go_import_path: github.com/apache/arrow
os: linux
go:
- 1.12.x
before_script:
- if [ $ARROW_CI_GO_AFFECTED != "1" ]; then exit; fi
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_go.sh
after_success:
- pushd ${TRAVIS_BUILD_DIR}/go/arrow
#- bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports"
# -------------------------------------------------------------------------
- name: R (with and without libarrow)
language: r
r: devel
cache:
directories:
- $HOME/.ccache
packages: true
dist: xenial
env:
- ARROW_TRAVIS_PARQUET=1
- ARROW_TRAVIS_PARQUET_ENCRYPTION=1
- ARROW_TRAVIS_USE_SYSTEM=1
- ARROW_TRAVIS_MIMALLOC=1
before_install:
# Have to copy-paste this here because of how R's build steps work
- eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py`
- if [ $ARROW_CI_R_AFFECTED != "1" ]; then exit; fi
- source ci/travis/before-install.sh
# First check that it builds without libarrow
- sudo apt-get install devscripts # For checkbashisms
- pushd ${TRAVIS_BUILD_DIR}/r
- R -e 'install.packages("remotes"); remotes::install_deps(dep = TRUE)'
- R CMD build .
- R CMD check --as-cran arrow_*tar.gz
- rm arrow_*tar.gz
- popd
# Now, proceed to install the c++ lib and the rest of the job
- |
if [ $TRAVIS_OS_NAME == "linux" ]; then
sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry"
sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
sudo apt-get update -qq
fi
- $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
# On Xenial, installing clang-7 leads these to get removed,
# but that breaks R, so let's reinstall them.
- sudo apt-get install gfortran gfortran-5 libblas-dev libgfortran-5-dev libgfortran3 liblapack-dev liblapack3
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
- export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib
- export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib/pkgconfig
- export CXX11FLAGS=-Wall
# Add this env var so we can assert in the tests that the library is installed correctly
- export TEST_R_WITH_ARROW=TRUE
- pushd ${TRAVIS_BUILD_DIR}/r
#after_success:
#- Rscript ../ci/travis_upload_r_coverage.R
after_failure:
- source ${TRAVIS_BUILD_DIR}/ci/travis/after-failure.sh
after_script:
- source ${TRAVIS_BUILD_DIR}/ci/travis/after-script.sh
env:
global:
- secure: "GcrPtsKUCgNY7HKYjWlHQo8SiFrShDvdZSU8t1m1FJrE+UfK0Dgh9zXmAausM8GmhqSwkF0q4UbLQf2uCnSITWKeEPAL8Mo9eu4ib+ikJx/b3Sk81frgW5ADoHfW1Eyqd8xJNIMwMegJOtRLSDqiXh1CvMlKnY8PyTOGM2DgN9ona/v6p9OFH9Qs0JhBRVXAn0S4ztjumck8E56+01hqRfxbZ88pTfpKghBxYp9PJaMjtGdomjVWlqPaWaWJj+KptT8inV9NK+TVYKx0dXWD+S1Vgr1PytQnLdILOYV23gsOBYqn33ByF/yADl4m3hUjU/qeT0Fi7aWxmVpj+oTJISOSH5N8nIsuNH8mQk2ZzzXHfV7btFvP+cOPRczadoKkT6D6cHA8nQ7b0dphC6bl6SAeSfc/cbhRT+fYnIjg8jFXC8jlyWBr7LR6GXVpc0bND7i300ITo0FuRJhy2OxqPtGo3dKLE7eAcv78tuO0OYJ/kol1PEqFdFkbYbNVbg/cFpbGqiCXDsOtPDbAGBv69YnXdVowSxxs8cRGjSkDydv6ZSytb/Zd4lH/KAomcFNk8adx12O1Lk4sbmVav1cGig5P6OcQKS0jC5IiRb4THcQzVzAkXXbaafKm5sru/NoYxhzmkyhkOc11nTYHKVng+XKWzLCNn7pTTSLitp5+xa4="
README.md
-1
View File
@@ -19,7 +19,6 @@
# Apache Arrow
[![Build Status](https://travis-ci.org/apache/arrow.svg?branch=master)](https://travis-ci.org/apache/arrow)
[![Build Status](https://ci.appveyor.com/api/projects/status/github/apache/arrow/branch/master?svg=true)](https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/branch/master)
[![Coverage Status](https://codecov.io/gh/apache/arrow/branch/master/graph/badge.svg)](https://codecov.io/gh/apache/arrow?branch=master)
[![Fuzzit Status](https://app.fuzzit.dev/badge?org_id=yMxZh42xl9qy6bvg3EiJ&branch=master)](https://app.fuzzit.dev/admin/yMxZh42xl9qy6bvg3EiJ/dashboard)
appveyor.yml
-16
View File
@@ -90,22 +90,6 @@ environment:
MINGW_PREFIX: c:\msys64\mingw64
MSYSTEM: MINGW64
USE_CLCACHE: false
- JOB: "Rust"
TARGET: x86_64-pc-windows-msvc
USE_CLCACHE: false
- JOB: "C#"
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
USE_CLCACHE: false
- JOB: "Go"
MINGW_PACKAGE_PREFIX: mingw-w64-x86_64
MINGW_PREFIX: c:\msys64\mingw64
MSYSTEM: MINGW64
USE_CLCACHE: false
GOROOT: 'c:\go112'
GOPATH: c:\gopath
GO111MODULE: on
GOTOOLDIR: '%GOROOT%\pkg\tool\windows_amd64'
PATH: '%GOPATH%\bin;%GOROOT%\bin;%PATH%'
- JOB: "R with libarrow"
USE_CLCACHE: false
TEST_R_WITH_ARROW: "TRUE"
ci/appveyor-build.bat
-16
View File
@@ -17,10 +17,6 @@
@echo on
IF /i "%JOB%" == "C#" goto csharp
IF /i "%JOB%" == "rust" goto rust
IF /i "%JOB%" == "go" goto golang
@rem All other builds are C++ builds
goto cpp
@@ -34,16 +30,4 @@ if "%JOB:~,5%" == "MinGW" (
)
goto scriptexit
:csharp
call ci\appveyor-csharp-build.bat
goto scriptexit
:rust
call ci\rust-build-main.bat
goto scriptexit
:golang
call ci\go-build-main.bat
goto scriptexit
:scriptexit
ci/conda_env_cpp.yml
+1
View File
@@ -30,6 +30,7 @@ grpc-cpp>=1.21.4
gtest=1.8.1
libprotobuf
lz4-c
make
ninja
pkg-config
python
ci/conda_env_python.yml
+1 -1
View File
@@ -19,10 +19,10 @@ cython=0.29.7
cloudpickle
hypothesis
numpy>=1.14
pandas
pytest
pytest-faulthandler
pytest-lazy-fixture
pytz
setuptools
setuptools_scm=3.2.0
pickle5
cpp/Dockerfile → ci/docker/conda-latest-cpp.dockerfile
+35 -26
View File
@@ -15,48 +15,57 @@
# specific language governing permissions and limitations
# under the License.
FROM ubuntu:18.04
ARG arch=amd64
FROM ${arch}/ubuntu:18.04
# arch is unset after the FROM statement, so need to define it again
ARG arch=amd64
ARG conda=latest
ARG minio=latest
ARG prefix=/opt/conda
# install build essentials
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
ca-certificates \
ccache \
g++ \
gcc \
git \
ninja-build \
pkg-config \
tzdata \
wget \
apt-get install -y -q wget tzdata \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# install conda and required packages
ARG EXTRA_CONDA_PKGS
ENV PATH=/opt/conda/bin:$PATH \
CONDA_PREFIX=/opt/conda
COPY ci/docker_install_conda.sh \
ci/conda_env_cpp.yml \
ENV PATH=${prefix}/bin:$PATH \
CONDA_PREFIX=${prefix}
# install conda and minio
COPY ci/scripts/install_conda.sh \
ci/scripts/install_minio.sh \
/arrow/ci/scripts/
RUN /arrow/ci/scripts/install_conda.sh ${arch} linux ${minio} ${prefix} && \
/arrow/ci/scripts/install_minio.sh ${arch} linux ${conda} ${prefix}
# install the required conda packages
COPY ci/conda_env_cpp.yml \
ci/conda_env_gandiva.yml \
ci/conda_env_unix.yml \
/arrow/ci/
RUN arrow/ci/docker_install_conda.sh && \
conda install -q -c conda-forge \
RUN conda install -q \
--file arrow/ci/conda_env_unix.yml \
--file arrow/ci/conda_env_cpp.yml \
--file arrow/ci/conda_env_gandiva.yml \
--file arrow/ci/conda_env_unix.yml \
$EXTRA_CONDA_PKGS && \
git compilers && \
conda clean --all
# TODO(kszucs): compilers instead of conda_env_linux
ENV CC=gcc \
CXX=g++ \
ENV ARROW_S3=ON \
ARROW_ORC=ON \
ARROW_GANDIVA=ON \
ARROW_PARQUET=ON \
ARROW_PLASMA=ON \
ARROW_FLIGHT=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_ZSTD=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_BROTLI=ON \
ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=CONDA \
ARROW_HOME=$CONDA_PREFIX \
PARQUET_HOME=$CONDA_PREFIX
# build and test
CMD ["arrow/ci/docker_build_and_test_cpp.sh"]
integration/dask/Dockerfile → ci/docker/conda-latest-integration.dockerfile
+32 -7
View File
@@ -15,13 +15,38 @@
# specific language governing permissions and limitations
# under the License.
FROM arrowdev/arrow-python-3.6:latest
ARG org
ARG arch=amd64
ARG conda=latest
FROM ${org}/${arch}-conda-${conda}-cpp:latest
# install dask release from conda
RUN conda install -c conda-forge dask pytest=3 && \
ARG arch=amd64
ARG maven=3.5
ARG node=11
ARG jdk=8
ARG go=1.12
RUN conda install -q \
numpy \
maven=${maven} \
nodejs=${node} \
openjdk=${jdk} && \
conda clean --all
# build and test
CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && \
arrow/ci/docker_build_python.sh && \
arrow/integration/dask/runtest.sh"]
ENV GOROOT=/opt/go \
GOBIN=/opt/go/bin \
GOPATH=/go \
PATH=/opt/go/bin:$PATH
RUN wget -nv -O - https://dl.google.com/go/go${go}.linux-${arch}.tar.gz | tar -xzf - -C /opt
ENV ARROW_BUILD_INTEGRATION=ON \
ARROW_FLIGHT=ON \
ARROW_ORC=OFF \
ARROW_DATASET=OFF \
ARROW_GANDIVA=OFF \
ARROW_PLASMA=OFF \
ARROW_FILESYSTEM=OFF \
ARROW_HDFS=OFF \
ARROW_JEMALLOC=OFF \
ARROW_JSON=OFF \
ARROW_USE_GLOG=OFF
ci/travis_script_go.sh → ci/docker/conda-latest-python-dask.dockerfile
Executable → Regular
+12 -20
View File
@@ -1,5 +1,3 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,23 +15,17 @@
# specific language governing permissions and limitations
# under the License.
set -e
ARG org
ARG arch=amd64
ARG conda=latest
ARG python=3.6
FROM ${org}/${arch}-conda-${conda}-python-${python}:latest
GO_DIR=${TRAVIS_BUILD_DIR}/go/arrow
pushd $GO_DIR
go get -d -t -v ./...
go install -v ./...
echo "" > coverage.txt
for d in $(go list ./... | grep -v vendor); do
go test -race -coverprofile=profile.out -covermode=atomic $d
if [ -f profile.out ]; then
cat profile.out >> coverage.txt
rm profile.out
ARG dask=latest
RUN if [ "${dask}" = "master" ]; then \
pip install https://github.com/dask/dask/archive/master.zip; \
elif [ "${dask}" = "latest" ]; then \
conda install -q dask && conda clean --all; \
else \
conda install -q dask=${dask} && conda clean --all; \
fi
done
popd
docs/Dockerfile → ci/docker/conda-latest-python-hdfs.dockerfile
+26 -4
View File
@@ -15,10 +15,32 @@
# specific language governing permissions and limitations
# under the License.
FROM arrowdev/arrow-python-3.6:latest
ARG org
ARG arch=amd64
ARG conda=latest
ARG python=3.6
FROM ${org}/${arch}-conda-${conda}-python-${python}:latest
COPY ci/conda_env_sphinx.yml /arrow/ci/
RUN conda install --file arrow/ci/conda_env_sphinx.yml && \
ARG jdk=8
ARG maven=3.5
RUN conda install -q \
pandas \
openjdk=${jdk} \
maven=${maven} && \
conda clean --all
CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && arrow/ci/docker_build_python.sh && arrow/ci/docker_build_sphinx.sh"]
# installing libhdfs (JNI)
ARG hdfs=2.9.2
ENV HADOOP_HOME=/opt/hadoop-${hdfs} \
HADOOP_OPTS=-Djava.library.path=/opt/hadoop-${hdfs}/lib/native \
PATH=$PATH:/opt/hadoop-${hdfs}/bin:/opt/hadoop-${hdfs}/sbin
RUN wget -q -O - "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=hadoop/common/hadoop-${hdfs}/hadoop-${hdfs}.tar.gz" | tar -xzf - -C /opt
COPY ci/etc/hdfs-site.xml $HADOOP_HOME/etc/hadoop/
# build cpp with tests
ENV CC=gcc \
CXX=g++ \
ARROW_ORC=ON \
ARROW_HDFS=ON \
ARROW_PYTHON=ON \
ARROW_BUILD_TESTS=ON
ci/docker_build_and_test_cpp.sh → ci/docker/conda-latest-python-pandas.dockerfile
Executable → Regular
+13 -16
View File
@@ -1,4 +1,3 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,19 +15,17 @@
# specific language governing permissions and limitations
# under the License.
set -e
ARG org
ARG arch=amd64
ARG conda=latest
ARG python=3.6
FROM ${org}/${arch}-conda-${conda}-python-${python}:latest
/arrow/ci/docker_build_cpp.sh
pushd /build/cpp
export ARROW_TEST_DATA=/arrow/testing/data
export PARQUET_TEST_DATA=/arrow/cpp/submodules/parquet-testing/data
# ARROW-5653 after install target, RPATH is modified on linux. Add this such
# that libraries are found with conda.
if [[ ! -z "${CONDA_PREFIX}" ]]; then
export LD_LIBRARY_PATH=${CONDA_PREFIX}/lib
fi
ninja unittest
popd
ARG pandas=latest
RUN if [ "${pandas}" = "master" ]; then \
pip install https://github.com/pandas-dev/pandas/archive/master.zip; \
elif [ "${pandas}" = "latest" ]; then \
conda install -q pandas && conda clean --all; \
else \
conda install -q pandas=${pandas} && conda clean --all; \
fi
ci/docker/conda-latest-python-spark.dockerfile
+50
View File
@@ -0,0 +1,50 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
ARG org
ARG arch=amd64
ARG conda=latest
ARG python=3.6
FROM ${org}/${arch}-conda-${conda}-python-${python}:latest
ARG jdk=8
ARG maven=3.5
RUN conda install -q \
patch \
pandas \
openjdk=${jdk} \
maven=${maven} && \
conda clean --all
# ENV MAVEN_HOME=/usr/local/maven \
# M2_HOME=/root/.m2 \
# PATH=/root/.m2/bin:/usr/local/maven/bin:$PATH
# installing specific version of spark
ARG spark=master
RUN mkdir /spark && wget -q -O - https://github.com/apache/spark/archive/${spark}.tar.gz | tar -xzf - --strip-components=1 -C /spark
# patch spark to build with current Arrow Java
COPY ci/etc/ARROW-6429.patch /tmp/
RUN patch -d /spark -p1 -i /tmp/ARROW-6429.patch && \
rm /tmp/ARROW-6429.patch
# build cpp with tests
ENV CC=gcc \
CXX=g++ \
ARROW_PYTHON=ON \
ARROW_HDFS=ON \
ARROW_BUILD_TESTS=OFF
integration/turbodbc/Dockerfile → ci/docker/conda-latest-python-turbodbc.dockerfile
+29 -10
View File
@@ -15,24 +15,43 @@
# specific language governing permissions and limitations
# under the License.
FROM arrowdev/arrow-python-3.6:latest
ARG org
ARG arch=amd64
ARG conda=latest
ARG python=3.6
FROM ${org}/${arch}-conda-${conda}-python-${python}:latest
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
odbc-postgresql \
postgresql \
sudo \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
sudo && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# install turbodbc dependencies from conda-forge
RUN conda install -c conda-forge pybind11 pytest pytest-cov mock unixodbc && \
RUN conda install -c conda-forge \
pybind11 \
pytest-cov \
mock \
unixodbc && \
conda clean --all
ENV TURBODBC_TEST_CONFIGURATION_FILES "query_fixtures_postgresql.json"
RUN service postgresql start && \
sudo -u postgres psql -U postgres -c \
"CREATE DATABASE test_db;" && \
sudo -u postgres psql -U postgres -c \
"ALTER USER postgres WITH PASSWORD 'password';"
# build and test
CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && \
arrow/ci/docker_build_python.sh && \
arrow/integration/turbodbc/runtest.sh"]
ARG turbodbc=latest
RUN git clone --recurse-submodules https://github.com/blue-yonder/turbodbc /turbodbc && \
if [ "${turbodbc}" = "master" ]; then \
git -C /turbodbc checkout master; \
elif [ "${turbodbc}" = "latest" ]; then \
git -C /turbodbc checkout $(git describe --tags); \
else \
git -C /turbodbc checkout ${turbodbc}; \
fi
ENV TURBODBC_TEST_CONFIGURATION_FILES "query_fixtures_postgresql.json"
python/Dockerfile → ci/docker/conda-latest-python.dockerfile
+8 -6
View File
@@ -15,19 +15,21 @@
# specific language governing permissions and limitations
# under the License.
FROM arrowdev/arrow-cpp:latest
ARG org
ARG arch
ARG conda
FROM ${org}/${arch}-conda-${conda}-cpp:latest
# install python specific packages
ARG PYTHON_VERSION=3.6
ARG python=3.6
COPY ci/conda_env_python.yml /arrow/ci/
RUN conda install -q \
--file arrow/ci/conda_env_python.yml \
python=$PYTHON_VERSION \
tensorflow \
nomkl && \
conda clean --all
ENV ARROW_PYTHON=ON \
ARROW_BUILD_TESTS=OFF
# build and test
CMD ["arrow/ci/docker_build_and_test_python.sh"]
ARROW_BUILD_TESTS=OFF \
ARROW_TENSORFLOW=ON
r/Dockerfile.conda → ci/docker/conda-latest-r.dockerfile
+8 -16
View File
@@ -15,23 +15,23 @@
# specific language governing permissions and limitations
# under the License.
FROM arrowdev/arrow-cpp:latest
ARG org
ARG arch
ARG conda
FROM ${org}/${arch}-conda-${conda}-cpp:latest
# install R specific packages
ARG R_VERSION=3.6.1
ARG r=3.6.1
COPY ci/conda_env_r.yml /arrow/ci/
RUN conda install -q \
--file arrow/ci/conda_env_r.yml \
r-base=$R_VERSION \
r-base=$r \
nomkl && \
conda clean --all
# Ensure parallel compilation of each individual package
RUN printf "\nMAKEFLAGS=-j8\n" >> /opt/conda/lib/R/etc/Makeconf
ENV MAKEFLAGS=-j8
ENV R_CONDA=1
# Arrow build flags
ENV ARROW_FLIGHT=OFF \
ARROW_GANDIVA=OFF \
@@ -42,13 +42,5 @@ ENV ARROW_FLIGHT=OFF \
ARROW_USE_ASAN=OFF \
ARROW_USE_UBSAN=OFF \
ARROW_NO_DEPRECATED_API=ON \
ARROW_R_DEV=TRUE
ENV ARROW_BUILD_BENCHMARKS=off
ENV ARROW_BUILD_EXAMPLES=off
ENV ARROW_BUILD_TESTS=off
ENV ARROW_BUILD_UTILITIES=off
# build, install, test R package
CMD ["/bin/bash", "-c", "/arrow/ci/docker_build_cpp.sh && \
/arrow/ci/docker_build_r.sh"]
ARROW_R_DEV=TRUE \
ARROW_BUILD_TESTS=OFF
cpp/Dockerfile.ubuntu-bionic → ci/docker/cuda-10.0-cpp.dockerfile
+7 -37
View File
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
FROM ubuntu:bionic
FROM nvidia/cuda:10.0-devel-ubuntu18.04
# pipefail is enabled for proper error detection in the `wget | apt-key add`
# step
@@ -28,26 +28,6 @@ RUN apt-get update -y -q && \
wget software-properties-common gpg-agent && \
apt-get clean && rm -rf /var/lib/apt/lists*
# Installs LLVM toolchain, for gandiva and testing other compilers
#
# Note that this is installed before the base packages to improve iteration
# while debugging package list with docker build due to slow download speed of
# llvm compared to ubuntu apt mirrors.
ARG LLVM_VERSION=7
# Args are only exposed in the "build" step, this ensure that LLVM_VERSION is
# found in the "run" step.
ENV LLVM_VERSION=${LLVM_VERSION}
ARG LLVM_APT_URL="http://apt.llvm.org/bionic/"
ARG LLVM_APT_ARCH="llvm-toolchain-bionic-${LLVM_VERSION}"
RUN wget -q -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
apt-add-repository -y --update "deb ${LLVM_APT_URL} ${LLVM_APT_ARCH} main" && \
apt-get install -y -q --no-install-recommends \
clang-${LLVM_VERSION} \
clang-format-${LLVM_VERSION} \
clang-tidy-${LLVM_VERSION} \
llvm-${LLVM_VERSION}-dev && \
apt-get clean && rm -rf /var/lib/apt/lists*
# Installs C++ toolchain and dependencies
RUN apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
@@ -85,6 +65,7 @@ RUN apt-get update -y -q && \
tzdata && \
apt-get clean && rm -rf /var/lib/apt/lists*
# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
# provided by the distribution:
# - libc-ares-dev does not install CMake config files
@@ -92,26 +73,15 @@ RUN apt-get update -y -q && \
# - libgtest-dev only provide sources
# - libprotobuf-dev only provide sources
# - thrift is too old
ENV CMAKE_ARGS="-DThrift_SOURCE=BUNDLED \
-DGTest_SOURCE=BUNDLED \
-DORC_SOURCE=BUNDLED"
# Prioritize system packages and local installation
ENV ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_FLIGHT=OFF \
ARROW_GANDIVA=ON \
ARROW_HDFS=ON \
ARROW_ORC=ON \
ARROW_PARQUET=ON \
ARROW_CUDA=ON \
ARROW_PLASMA=ON \
ARROW_USE_ASAN=ON \
ARROW_USE_UBSAN=ON \
ARROW_NO_DEPRECATED_API=ON \
ARROW_INSTALL_NAME_RPATH=OFF \
ARROW_WITH_BZ2=ON \
ARROW_WITH_ZSTD=ON
ENV CC=clang-${LLVM_VERSION} \
CXX=clang++-${LLVM_VERSION}
CMD ["arrow/ci/docker_build_and_test_cpp.sh"]
ARROW_HOME=/usr/local \
ORC_SOURCE=BUNDLED \
Thrift_SOURCE=BUNDLED \
GTest_SOURCE=BUNDLED
cpp/Dockerfile.debian-testing → ci/docker/debian-10-cpp.dockerfile
+25 -22
View File
@@ -15,7 +15,8 @@
# specific language governing permissions and limitations
# under the License.
FROM debian:testing
ARG arch=amd64
FROM ${arch}/debian:10
# install build essentials
RUN export DEBIAN_FRONTEND=noninteractive && \
@@ -40,8 +41,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
libgoogle-glog-dev \
libgtest-dev \
liblz4-dev \
libprotobuf-dev \
libprotoc-dev \
libre2-dev \
libsnappy-dev \
libssl-dev \
@@ -50,29 +49,33 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
llvm-7-dev \
make \
ninja-build \
openjdk-11-jdk \
openjdk-11-jdk-headless \
pkg-config \
protobuf-compiler \
rapidjson-dev \
thrift-compiler \
tzdata \
wget \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
zlib1g-dev \
wget && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
ENV CC=gcc \
CXX=g++ \
ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_FLIGHT=ON \
ARROW_GANDIVA=ON \
ARROW_GANDIVA_JAVA=ON \
ARROW_HOME=/usr \
JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 \
CMAKE_ARGS="-Dc-ares_SOURCE=BUNDLED \
-DgRPC_SOURCE=BUNDLED \
-DORC_SOURCE=BUNDLED"
# build and test
CMD ["arrow/ci/docker_build_and_test_cpp.sh"]
CXX=g++ \
cares_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
gRPC_SOURCE=BUNDLED \
Protobuf_SOURCE=BUNDLED \
ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_FLIGHT=ON \
ARROW_PARQUET=ON \
ARROW_PLASMA=ON \
ARROW_ORC=ON \
ARROW_GANDIVA=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_ZSTD=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_BROTLI=ON \
ARROW_HOME=/usr/local
ci/docker/debian-10-go.dockerfile
+25
View File
@@ -0,0 +1,25 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
ARG arch=amd64
ARG go=1.12
FROM ${arch}/golang:${go}
# TODO(kszucs):
# 1. add the files required to install the dependencies to .dockeignore
# 2. copy these files to their appropiate path
# 3. download and compile the dependencies
ci/docker/debian-10-js.dockerfile
+27
View File
@@ -0,0 +1,27 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
ARG arch=amd64
ARG node=11
FROM ${arch}/node:${node}
ENV NODE_NO_WARNINGS=1
# TODO(kszucs):
# 1. add the files required to install the dependencies to .dockeignore
# 2. copy these files to their appropiate path
# 3. download and compile the dependencies
ci/docker/debian-10-rust.dockerfile
+44
View File
@@ -0,0 +1,44 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
ARG arch=amd64
FROM ${arch}/rust
# install pre-requisites for building flatbuffers
RUN apt-get update -y && \
apt-get install -y build-essential cmake && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# install flatbuffers
ARG flatbuffers=1.11.0
RUN wget -q -O - https://github.com/google/flatbuffers/archive/v${flatbuffers}.tar.gz | tar -xzf - && \
cd flatbuffers-${flatbuffers} && \
cmake -G "Unix Makefiles" && \
make install && \
cd / && \
rm -rf flatbuffers-${flatbuffers}
# sadly cargo doesn't have a command to fetch and build the
# dependencies without building the library itself
ARG rust=nightly-2019-09-25
RUN rustup default ${rust}
# TODO(kszucs):
# 1. add the files required to install the dependencies to .dockeignore
# 2. copy these files to their appropiate path
# 3. download and compile the dependencies
ci/docker/debian-9-java.dockerfile
+28
View File
@@ -0,0 +1,28 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
ARG arch=amd64
ARG jdk=8
ARG maven=3.5.4
FROM ${arch}/maven:${maven}-jdk-${jdk}
ENV ARROW_JAVA_SHADE_FLATBUFS=ON
# TODO(kszucs):
# 1. add the files required to install the dependencies to .dockeignore
# 2. copy these files to their appropiate path
# 3. download and compile the dependencies
cpp/Dockerfile.fedora → ci/docker/fedora-29-cpp.dockerfile
+29 -16
View File
@@ -15,7 +15,8 @@
# specific language governing permissions and limitations
# under the License.
FROM fedora:29
ARG arch
FROM ${arch}/fedora:29
# install dependencies
RUN dnf update -y && \
@@ -59,19 +60,31 @@ RUN dnf update -y && \
# when built via ExternalProject: https://bugzilla.redhat.com/show_bug.cgi?id=1687844
# * protobuf libraries in Fedora 29 are too old for gRPC
ENV CC=gcc \
CXX=g++ \
ARROW_ORC=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_PARQUET=ON \
ARROW_FLIGHT=ON \
ARROW_GANDIVA=ON \
ARROW_GANDIVA_JAVA=ON \
ARROW_BUILD_TESTS=ON \
ARROW_HOME=/usr/local \
CMAKE_ARGS="\
-Dc-ares_SOURCE=BUNDLED \
-DgRPC_SOURCE=BUNDLED \
-DORC_SOURCE=BUNDLED \
-DProtobuf_SOURCE=BUNDLED"
CXX=g++ \
ARROW_ORC=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
cares_SOURCE=BUNDLED \
gRPC_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
Protobuf_SOURCE=BUNDLED \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_ZSTD=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_BROTLI=ON \
ARROW_PARQUET=ON \
ARROW_FLIGHT=ON \
ARROW_GANDIVA=OFF \
ARROW_GANDIVA_JAVA=ON \
ARROW_BUILD_TESTS=ON \
ARROW_HOME=/usr/local
CMD ["arrow/ci/docker_build_and_test_cpp.sh"]
# Gandiva test is failing with:
# Running gandiva-internals-test, redirecting output into /build/cpp/build/test-logs/gandiva-internals-test.txt (attempt 1/1)
# 1364
# : CommandLine Error: Option 'x86-experimental-vector-widening-legalization' registered more than once!
# 1365
# LLVM ERROR: inconsistency in registered CommandLine options
# 1366
# /build/cpp/src/gandiva
ci/docker/linux-apt-c-glib.dockerfile
+44
View File
@@ -0,0 +1,44 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
ARG base
FROM ${base}
RUN apt-get update -y -q && \
apt-get install -y -q \
python3 \
python3-pip \
gtk-doc-tools \
libgirepository1.0-dev \
libglib2.0-doc \
luarocks \
pkg-config \
ruby-dev && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
RUN luarocks install lgi
RUN pip3 install meson && \
gem install bundler
COPY c_glib/Gemfile /arrow/c_glib/
RUN bundle install --gemfile /arrow/c_glib/Gemfile
ENV ARROW_BUILD_TESTS=OFF \
ARROW_BUILD_UTILITIES=OFF \
ARROW_INSTALL_NAME_RPATH=OFF
dev/fuzzit/Dockerfile → ci/docker/linux-apt-cmake.dockerfile
+8 -7
View File
@@ -15,14 +15,15 @@
# specific language governing permissions and limitations
# under the License.
FROM arrowdev/arrow-cpp:latest
ARG base
FROM ${base}
RUN apt-get update && \
apt-get install -y -q \
clang-7 \
libclang-7-dev \
clang-format-7 \
clang-tidy-7 \
clang-tools-7
libidn11 \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
CMD ["arrow/ci/docker_build_and_fuzzit.sh"]
ARG cmake=3.2.3
RUN wget -nv -O - https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-x86_64.tar.gz | tar -xzf - -C /opt
ENV PATH=/opt/cmake-${cmake}-Linux-x86_64/bin:$PATH
c_glib/Dockerfile → ci/docker/linux-apt-docs.dockerfile
+33 -24
View File
@@ -15,36 +15,45 @@
# specific language governing permissions and limitations
# under the License.
FROM arrowdev/arrow-cpp:latest
ARG base
FROM ${base}
ARG jdk=8
RUN apt-get update -y -q && \
apt-get -q install --no-install-recommends -y \
apt-get install -y -q --no-install-recommends \
autoconf-archive \
automake \
doxygen \
gobject-introspection \
gtk-doc-tools \
libgirepository1.0-dev \
libglib2.0-dev \
pkg-config \
libglib2.0-doc \
libtool \
ninja-build \
openjdk-${jdk}-jdk-headless \
ruby-dev \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
rsync && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
ENV JAVA_HOME=/usr/lib/jvm/java-${jdk}-openjdk-amd64
ARG maven=3.5.4
RUN wget -q -O - "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=maven/maven-3/${maven}/binaries/apache-maven-${maven}-bin.tar.gz" | tar -xzf - -C /opt
ENV PATH=/opt/apache-maven-${maven}/bin:$PATH
ARG node=11
RUN wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \
apt-get install -y nodejs && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
RUN pip install \
meson \
breathe \
ipython \
sphinx \
sphinx_rtd_theme
COPY c_glib/Gemfile /arrow/c_glib/
RUN conda install meson=0.50.1 && \
conda clean --all && \
gem install bundler && \
bundle install --gemfile arrow/c_glib/Gemfile
# build cpp
ENV ARROW_BUILD_TESTS=OFF \
ARROW_BUILD_UTILITIES=OFF \
ARROW_INSTALL_NAME_RPATH=OFF \
LD_LIBRARY_PATH="${CONDA_PREFIX}/lib" \
PKG_CONFIG=/usr/bin/pkg-config \
PKG_CONFIG_PATH="${CONDA_PREFIX}/lib/pkgconfig" \
GI_TYPELIB_PATH="${CONDA_PREFIX}/lib/girepository-1.0"
# build, install and test
CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && \
arrow/ci/docker_build_c_glib.sh && \
arrow/c_glib/test/run-test.rb"]
RUN gem install bundler && \
bundle install --gemfile /arrow/c_glib/Gemfile
ci/docker/linux-apt-fuzzit.dockerfile
+47
View File
@@ -0,0 +1,47 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
ARG base
FROM ${base}
RUN apt-get update && \
apt-get install -y -q \
clang-7 \
libclang-7-dev \
clang-format-7 \
clang-tidy-7 \
clang-tools-7 && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
RUN wget -O /usr/local/bin/fuzzit https://github.com/fuzzitdev/fuzzit/releases/latest/download/fuzzit_Linux_x86_64 && \
chmod a+x /usr/local/bin/fuzzit
ENV ARROW_FUZZING="ON" \
ARROW_USE_ASAN="ON" \
CC="clang-7" \
CXX="clang++-7" \
ARROW_BUILD_TYPE="RelWithDebInfo" \
ARROW_FLIGHT="OFF" \
ARROW_GANDIVA="OFF" \
ARROW_ORC="OFF" \
ARROW_PARQUET="OFF" \
ARROW_PLASMA="OFF" \
ARROW_WITH_BZ2="OFF" \
ARROW_WITH_ZSTD="OFF" \
ARROW_BUILD_BENCHMARKS="OFF" \
ARROW_BUILD_UTILITIES="OFF"
cpp/Dockerfile.cmake → ci/docker/linux-apt-jni.dockerfile
+37 -35
View File
@@ -15,54 +15,56 @@
# specific language governing permissions and limitations
# under the License.
FROM ubuntu:18.04
ARG base
FROM ${base}
# install build essentials
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
ca-certificates \
curl \
ccache \
clang-7 \
cmake \
git \
g++ \
gcc \
git \
libidn11 \
libboost-all-dev \
libdouble-conversion-dev \
libgflags-dev \
libgoogle-glog-dev \
libgtest-dev \
liblz4-dev \
libre2-dev \
libsnappy-dev \
libssl-dev \
llvm-7-dev \
make \
ninja-build \
pkg-config \
protobuf-compiler \
rapidjson-dev \
thrift-compiler \
tzdata \
wget \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
zlib1g-dev \
wget && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# install conda and required packages
ARG EXTRA_CONDA_PKGS
ENV PATH=/opt/conda/bin:$PATH \
CONDA_PREFIX=/opt/conda
COPY ci/docker_install_conda.sh \
ci/conda_env_cpp.yml \
ci/conda_env_unix.yml \
/arrow/ci/
RUN arrow/ci/docker_install_conda.sh && \
conda install -q -c conda-forge \
--file arrow/ci/conda_env_cpp.yml \
--file arrow/ci/conda_env_unix.yml \
$EXTRA_CONDA_PKGS && \
conda clean --all
ARG CMAKE_VERSION=3.2.3
ARG CMAKE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
RUN curl ${CMAKE_URL} -Lo cmake-${CMAKE_VERSION}.tar.gz && \
mkdir /opt/cmake && tar -zxvf cmake-${CMAKE_VERSION}.tar.gz -C /opt/cmake
ENV PATH=/opt/cmake/cmake-${CMAKE_VERSION}-Linux-x86_64/bin:$PATH
ARG cmake=3.11.4
RUN wget -nv -O - https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-x86_64.tar.gz | tar -xzf - -C /opt
ENV PATH=/opt/cmake-${cmake}-Linux-x86_64/bin:$PATH
ENV CC=gcc \
CXX=g++ \
ARROW_GANDIVA=OFF \
ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=CONDA \
ARROW_HOME=$CONDA_PREFIX \
PARQUET_HOME=$CONDA_PREFIX
# build and test
CMD ["arrow/ci/docker_build_and_test_cpp.sh"]
ORC_SOURCE=BUNDLED \
Protobuf_SOURCE=BUNDLED \
ARROW_BUILD_TESTS=OFF \
ARROW_FLIGHT=OFF \
ARROW_PARQUET=OFF \
ARROW_PLASMA=ON \
ARROW_ORC=ON \
ARROW_JNI=ON \
ARROW_GANDIVA=ON \
ARROW_GANDIVA_JAVA=ON \
ARROW_HOME=/usr/local
ci/docker/Dockerfile.lint → ci/docker/linux-apt-lint.dockerfile
+35 -31
View File
@@ -15,43 +15,47 @@
# specific language governing permissions and limitations
# under the License.
ARG base
FROM hadolint/hadolint:v1.17.2 AS hadolint
FROM ubuntu:18.04
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
FROM ${base}
RUN apt-get update && \
apt-get install -y -q \
build-essential \
clang-7 \
clang-format-7 \
clang-tidy-7 \
clang-tools-7 \
cmake \
curl \
git \
libclang-7-dev \
ninja-build \
openjdk-11-jdk-headless \
python3 \
python3-dev \
python3-pip \
ruby \
apt-get install -y -q \
clang-7 \
clang-format-7 \
clang-tidy-7 \
clang-tools-7 \
cmake \
curl \
libclang-7-dev \
openjdk-11-jdk-headless \
python3 \
python3-dev \
python3-pip \
ruby \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Use python3 by default in scripts
RUN ln -s /usr/bin/python3 /usr/local/bin/python
RUN pip3 install flake8 cmake_format==0.5.2
# Rust linter
RUN curl https://sh.rustup.rs -sSf | \
sh -s -- --default-toolchain stable -y
ENV PATH /root/.cargo/bin:$PATH
RUN rustup component add rustfmt
# Docker linter
COPY --from=hadolint /bin/hadolint /usr/bin/hadolint
CMD ["/arrow/ci/docker/lint.sh"]
# IWYU
COPY ci/scripts/install_iwyu.sh /arrow/ci/scripts/
RUN arrow/ci/scripts/install_iwyu.sh /tmp/iwyu /usr/local 7
# Rust linter
ARG rust=nightly-2019-09-25
RUN curl https://sh.rustup.rs -sSf | \
sh -s -- --default-toolchain stable -y
ENV PATH /root/.cargo/bin:$PATH
RUN rustup install ${rust} && \
rustup default ${rust} && \
rustup component add rustfmt
# Use python3 by default in scripts
RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
ln -s /usr/bin/pip3 /usr/local/bin/pip
RUN pip install flake8 cmake_format==0.5.2 click
ENV LC_ALL=C.UTF-8 \
LANG=C.UTF-8
dev/lint/Dockerfile → ci/docker/linux-apt-python-3.dockerfile
+22 -12
View File
@@ -15,19 +15,29 @@
# specific language governing permissions and limitations
# under the License.
FROM arrowdev/arrow-python-3.6
ARG base
FROM ${base}
RUN apt-get update && \
RUN apt-get update -y -q && \
apt-get install -y -q \
clang-7 \
libclang-7-dev \
clang-format-7 \
clang-tidy-7 \
clang-tools-7
python3 \
python3-pip \
python3-dev && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
RUN conda install flake8 && \
conda clean --all -y
RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
ln -s /usr/bin/pip3 /usr/local/bin/pip
COPY python/requirements.txt \
python/requirements-test.txt \
/arrow/python/
RUN pip install \
-r arrow/python/requirements.txt \
-r arrow/python/requirements-test.txt \
cython setuptools
ENV ARROW_PYTHON=ON \
ARROW_BUILD_TESTS=OFF
ENV PATH=/opt/iwyu/bin:$PATH
ADD ci/docker_install_iwyu.sh /arrow/ci/
RUN arrow/ci/docker_install_iwyu.sh
ci/docker/linux-apt-r.dockerfile
+67
View File
@@ -0,0 +1,67 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
ARG base
FROM ${base}
# Build R
# [1] https://www.digitalocean.com/community/tutorials/how-to-install-r-on-ubuntu-18-04
# [2] https://linuxize.com/post/how-to-install-r-on-ubuntu-18-04/#installing-r-packages-from-cran
ARG r=3.6
RUN apt-get update -y && \
apt-get install -y \
dirmngr \
apt-transport-https \
software-properties-common && \
apt-key adv \
--keyserver keyserver.ubuntu.com \
--recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \
add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' && \
apt-get install -y \
r-base=${r}* \
# system libs needed by core R packages
libxml2-dev \
libgit2-dev \
libssl-dev \
# install clang to mirror what was done on Travis
clang \
clang-format \
clang-tidy \
# R CMD CHECK --as-cran needs pdflatex to build the package manual
texlive-latex-base && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Ensure parallel R package installation, set CRAN repo mirror,
# and use pre-built binaries where possible
RUN printf "\
options(Ncpus = parallel::detectCores(), \
repos = 'https://demo.rstudiopm.com/all/__linux__/bionic/latest', \
HTTPUserAgent = sprintf(\
'R/%%s R (%%s)', getRversion(), \
paste(getRversion(), R.version\$platform, R.version\$arch, R.version\$os)))\n" \
>> /etc/R/Rprofile.site
# Also ensure parallel compilation of each individual package
RUN echo "MAKEFLAGS=-j8" >> /usr/lib/R/etc/Makeconf
ENV ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_PARQUET=ON \
ARROW_USE_ASAN=OFF \
ARROW_USE_UBSAN=OFF \
ARROW_NO_DEPRECATED_API=ON \
ARROW_R_DEV=TRUE
ci/docker/lint.sh → ci/docker/linux-apt-ruby.dockerfile
Executable → Regular
+6 -12
View File
@@ -1,4 +1,3 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,15 +15,10 @@
# specific language governing permissions and limitations
# under the License.
set -eu
# depends on a C Glib image
ARG base
FROM ${base}
: ${ARROW_HOME:=/arrow}
install_archery() {
pip3 install -e ${ARROW_HOME}/dev/archery
export LC_ALL=C.UTF-8
export LANG=C.UTF-8
}
install_archery
archery lint
# TODO(kszucs): we should install the required dependencies here
# COPY ruby/ /arrow/
# RUN bundle install --gemfile /arrow/ruby/red-arrow/Gemfile
ci/docker/linux-dnf-python-3.dockerfile
+39
View File
@@ -0,0 +1,39 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
ARG base
FROM ${base}
RUN dnf install -y \
python3 \
python3-pip \
python3-devel
RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
ln -s /usr/bin/pip3 /usr/local/bin/pip
COPY python/requirements.txt \
python/requirements-test.txt \
/arrow/python/
RUN pip install \
-r arrow/python/requirements.txt \
-r arrow/python/requirements-test.txt \
cython
ENV ARROW_PYTHON=ON \
ARROW_BUILD_TESTS=OFF
cpp/Dockerfile.ubuntu-trusty → ci/docker/ubuntu-14.04-cpp.dockerfile
+37 -33
View File
@@ -15,7 +15,8 @@
# specific language governing permissions and limitations
# under the License.
FROM ubuntu:trusty
ARG arch=amd64
FROM ${arch}/ubuntu:14.04
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -41,11 +42,11 @@ RUN apt-get update -y -q && \
pkg-config \
protobuf-compiler \
python-pip \
tzdata \
&& apt-get clean \
&& python -m pip install --upgrade pip \
&& pip install --upgrade cmake \
&& rm -rf /var/lib/apt/lists/*
tzdata && \
apt-get clean && \
python -m pip install --upgrade pip && \
pip install --upgrade cmake && \
rm -rf /var/lib/apt/lists/*
# - Flight is deactivated because the OpenSSL in Ubuntu 14.04 is too old
# - Disabling Gandiva since LLVM 7 require gcc >= 4.9 toolchain
@@ -57,30 +58,33 @@ RUN apt-get update -y -q && \
# - libre2-dev unavailable
# - liblz4-dev is too old
ENV CC=gcc \
CXX=g++ \
ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_FLIGHT=OFF \
ARROW_WITH_OPENSSL=OFF \
ARROW_GANDIVA=OFF \
ARROW_GANDIVA_JAVA=OFF \
ARROW_ORC=OFF \
ARROW_PARQUET=ON \
ARROW_HOME=/usr \
CMAKE_ARGS="-DThrift_SOURCE=BUNDLED \
-DBOOST_SOURCE=BUNDLED \
-Dbenchmark_SOURCE=BUNDLED \
-DBrotli_SOURCE=BUNDLED \
-Dc-ares_SOURCE=BUNDLED \
-Dgflags_SOURCE=BUNDLED \
-DGTest_SOURCE=BUNDLED \
-DgRPC_SOURCE=BUNDLED \
-DLz4_SOURCE=BUNDLED \
-DORC_SOURCE=BUNDLED \
-DProtobuf_SOURCE=BUNDLED \
-DRapidJSON_SOURCE=BUNDLED \
-DRE2_SOURCE=BUNDLED \
-DZSTD_SOURCE=BUNDLED"
# build and test
CMD ["arrow/ci/docker_build_and_test_cpp.sh"]
CXX=g++ \
ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
Thrift_SOURCE=BUNDLED \
BOOST_SOURCE=BUNDLED \
benchmark_SOURCE=BUNDLED \
Brotli_SOURCE=BUNDLED \
RE2_SOURCE=BUNDLED \
ZSTD_SOURCE=BUNDLED \
Lz4_SOURCE=BUNDLED \
cares_SOURCE=BUNDLED \
gflags_SOURCE=BUNDLED \
GTest_SOURCE=BUNDLED \
gRPC_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
Protobuf_SOURCE=BUNDLED \
RapidJSON_SOURCE=BUNDLED \
ARROW_FLIGHT=OFF \
ARROW_WITH_OPENSSL=OFF \
ARROW_GANDIVA=OFF \
ARROW_GANDIVA_JAVA=OFF \
ARROW_ORC=OFF \
ARROW_PARQUET=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_ZSTD=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_BROTLI=ON \
ARROW_HOME=/usr/local
cpp/Dockerfile.ubuntu-xenial → ci/docker/ubuntu-16.04-cpp.dockerfile
+29 -28
View File
@@ -15,7 +15,8 @@
# specific language governing permissions and limitations
# under the License.
FROM ubuntu:xenial
ARG arch=amd64
FROM ${arch}/ubuntu:16.04
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -45,16 +46,15 @@ RUN apt-get update -y -q && \
libre2-dev \
libsnappy-dev \
libssl-dev \
libzstd-dev \
llvm-7-dev \
make \
ninja-build \
pkg-config \
protobuf-compiler \
python \
tzdata \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
python3 \
tzdata && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Benchmark is deactivated as the external project requires CMake 3.6+
# Gandiva JNI is deactivated as it requires CMake 3.11+
@@ -64,25 +64,26 @@ RUN apt-get update -y -q && \
# - libboost-all-dev does not include Boost.Process, needed for Flight
# unit tests, so doing vendored build by default
# - liburiparser-dev is too old (< 0.9.0)
ENV CC=gcc \
CXX=g++ \
ARROW_BUILD_BENCHMARKS=OFF \
ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_GANDIVA=ON \
ARROW_GANDIVA_JAVA=OFF \
ARROW_PARQUET=ON \
ARROW_HOME=/usr \
ARROW_WITH_ZSTD=OFF \
CMAKE_ARGS="-DThrift_SOURCE=BUNDLED \
-DBOOST_SOURCE=BUNDLED \
-Dc-ares_SOURCE=BUNDLED \
-Dgflags_SOURCE=BUNDLED \
-DgRPC_SOURCE=BUNDLED \
-DGTest_SOURCE=BUNDLED \
-DORC_SOURCE=BUNDLED \
-DProtobuf_SOURCE=BUNDLED \
-DRapidJSON_SOURCE=BUNDLED"
# build and test
CMD ["arrow/ci/docker_build_and_test_cpp.sh"]
ENV ARROW_BUILD_BENCHMARKS=OFF \
ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
BOOST_SOURCE=BUNDLED \
Thrift_SOURCE=BUNDLED \
cares_SOURCE=BUNDLED \
gRPC_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
GTest_SOURCE=BUNDLED \
Protobuf_SOURCE=BUNDLED \
RapidJSON_SOURCE=BUNDLED \
ARROW_GANDIVA_JAVA=OFF \
ARROW_GANDIVA=ON \
ARROW_HOME=/usr/local \
ARROW_PARQUET=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_ZSTD=OFF \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_BROTLI=ON \
CXX=g++ \
CC=gcc
ci/docker/ubuntu-18.04-cpp.dockerfile
+119
View File
@@ -0,0 +1,119 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
ARG arch=amd64
FROM ${arch}/ubuntu:18.04
# pipefail is enabled for proper error detection in the `wget | apt-key add`
# step
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
wget \
software-properties-common \
gpg-agent && \
apt-get clean && \
rm -rf /var/lib/apt/lists*
# Installs LLVM toolchain, for gandiva and testing other compilers
#
# Note that this is installed before the base packages to improve iteration
# while debugging package list with docker build due to slow download speed of
# llvm compared to ubuntu apt mirrors.
ARG llvm_version=7
ARG llvm_apt_url="http://apt.llvm.org/bionic/"
ARG llvm_apt_arch="llvm-toolchain-bionic-${llvm_version}"
RUN wget -q -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
apt-add-repository -y --update "deb ${llvm_apt_url} ${llvm_apt_arch} main" && \
apt-get install -y -q --no-install-recommends \
clang-${llvm_version} \
clang-format-${llvm_version} \
clang-tidy-${llvm_version} \
llvm-${llvm_version}-dev && \
apt-get clean && \
rm -rf /var/lib/apt/lists*
# Installs C++ toolchain and dependencies
RUN apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
autoconf \
bison \
ca-certificates \
ccache \
cmake \
flex \
g++ \
gcc \
git \
libbenchmark-dev \
libboost-filesystem-dev \
libboost-regex-dev \
libboost-system-dev \
libbrotli-dev \
libbz2-dev \
libdouble-conversion-dev \
libgflags-dev \
libgoogle-glog-dev \
liblz4-dev \
libprotobuf-dev \
libprotoc-dev \
libre2-dev \
libsnappy-dev \
libssl-dev \
libzstd-dev \
ninja-build \
pkg-config \
protobuf-compiler \
rapidjson-dev \
thrift-compiler \
tzdata && \
apt-get clean && \
rm -rf /var/lib/apt/lists*
# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
# provided by the distribution:
# - libc-ares-dev does not install CMake config files
# - flatbuffer is not packaged
# - libgtest-dev only provide sources
# - libprotobuf-dev only provide sources
# - thrift is too old
ENV ARROW_BUILD_TESTS=ON \
ARROW_FLIGHT=OFF \
ARROW_GANDIVA=ON \
ARROW_HDFS=ON \
ARROW_HOME=/usr/local \
ARROW_INSTALL_NAME_RPATH=OFF \
ARROW_NO_DEPRECATED_API=ON \
ARROW_ORC=ON \
ARROW_PARQUET=ON \
ARROW_PLASMA=ON \
ARROW_USE_ASAN=OFF \
ARROW_USE_UBSAN=OFF \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_ZSTD=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_BROTLI=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ORC_SOURCE=BUNDLED \
GTest_SOURCE=BUNDLED \
Thrift_SOURCE=BUNDLED
ci/docker/ubuntu-18.04-csharp.dockerfile
+20
View File
@@ -0,0 +1,20 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
ARG platform=bionic
ARG dotnet=2.1
FROM mcr.microsoft.com/dotnet/core/sdk:${dotnet}-${platform}
r/Dockerfile.sanitizer → ci/docker/ubuntu-18.04-r-sanitizer.dockerfile
+13 -17
View File
@@ -70,6 +70,7 @@ ENV MAKEFLAGS=-j8
RUN RDsan -e "install.packages(c('remotes', 'dplyr', 'glue'))"
RUN RDsan -e "remotes::install_github('romainfrancois/decor')"
# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
# provided by the distribution:
# - libc-ares-dev does not install CMake config files
@@ -77,15 +78,12 @@ RUN RDsan -e "remotes::install_github('romainfrancois/decor')"
# - libgtest-dev only provide sources
# - libprotobuf-dev only provide sources
# - thrift is too old
ENV CMAKE_ARGS="-DThrift_SOURCE=BUNDLED \
-DFlatbuffers_SOURCE=BUNDLED \
-DGTest_SOURCE=BUNDLED \
-Dc-ares_SOURCE=BUNDLED \
-DgRPC_SOURCE=BUNDLED \
-DProtobuf_SOURCE=BUNDLED ${CMAKE_ARGS}"
# Prioritize system packages and local installation
ENV ARROW_DEPENDENCY_SOURCE=SYSTEM \
ENV gRPC_SOURCE=BUNDLED \
Protobuf_SOURCE=BUNDLED \
cares_SOURCE=BUNDLED \
GTest_SOURCE=BUNDLED \
Thrift_SOURCE=BUNDLED \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_FLIGHT=OFF \
ARROW_GANDIVA=OFF \
ARROW_HDFS=OFF \
@@ -98,12 +96,10 @@ ENV ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_INSTALL_NAME_RPATH=OFF \
ARROW_WITH_BZ2=OFF \
ARROW_WITH_ZSTD=OFF \
ARROW_R_DEV=TRUE
ARROW_R_DEV=TRUE \
R_BIN=RDsan
ENV ARROW_BUILD_BENCHMARKS=off
ENV ARROW_BUILD_EXAMPLES=off
ENV ARROW_BUILD_TESTS=off
ENV ARROW_BUILD_UTILITIES=off
CMD ["/bin/bash", "-c", "/arrow/ci/docker_build_cpp.sh && \
/arrow/ci/docker_build_r_sanitizer.sh"]
# Ensure parallel R package installation and set CRAN repo mirror
RUN printf "options(Ncpus = parallel::detectCores(), repos = 'https://demo.rstudiopm.com/all/__linux__/bionic/latest')\n" >> /etc/R/Rprofile.site
# Also ensure parallel compilation of each individual package
RUN printf "MAKEFLAGS=-j8\n" >> /usr/lib/R/etc/Makeconf
ci/docker_build_and_fuzzit.sh
-52
View File
@@ -1,52 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euxo pipefail
export ARROW_FUZZING="ON"
#export ARROW_DEPENDENCY_SOURCE="BUNDLED"
export ARROW_USE_ASAN="ON"
export CC="clang-7"
export CXX="clang++-7"
export ARROW_BUILD_TYPE="RelWithDebInfo"
export ARROW_FLIGHT="OFF"
export ARROW_GANDIVA="OFF"
export ARROW_ORC="OFF"
export ARROW_PARQUET="OFF"
export ARROW_PLASMA="OFF"
export ARROW_WITH_BZ2="OFF"
export ARROW_WITH_ZSTD="OFF"
export ARROW_BUILD_BENCHMARKS="OFF"
export ARROW_BUILD_UTILITIES="OFF"
/arrow/ci/docker_build_cpp.sh || exit 1
pushd /build/cpp
mkdir ./relwithdebinfo/out
cp ./relwithdebinfo/arrow-ipc-fuzzing-test ./relwithdebinfo/out/fuzzer
ldd ./relwithdebinfo/arrow-ipc-fuzzing-test | grep "=> /" | awk '{print $3}' | xargs -I '{}' cp -v '{}' ./relwithdebinfo/out/.
cd ./relwithdebinfo/out/
tar -czvf fuzzer.tar.gz *
stat fuzzer.tar.gz
cd ../../
export TARGET_ID=apache-arrow/arrow-ipc-fuzzing
wget -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/latest/download/fuzzit_Linux_x86_64
chmod a+x fuzzit
./fuzzit create job --type $FUZZIT_JOB_TYPE --host bionic-llvm7 --revision $CI_ARROW_SHA --branch $CI_ARROW_BRANCH $TARGET_ID ./relwithdebinfo/out/fuzzer.tar.gz
ci/docker_build_cpp.sh
-82
View File
@@ -1,82 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -ex
source_dir=${1:-/arrow/cpp}
build_dir=${2:-/build/cpp}
install_dir=${3:-${ARROW_HOME:-/usr/local}}
export CCACHE_COMPRESS=1
export CCACHE_COMPRESSLEVEL=5
export CCACHE_COMPILERCHECK=content
export CCACHE_DIR=/build/ccache
export PATH=/usr/lib/ccache/:$PATH
rm -rf ${build_dir}
mkdir -p ${build_dir}
pushd ${build_dir}
cmake -GNinja \
-DCMAKE_INSTALL_PREFIX=${install_dir} \
-DCMAKE_INSTALL_LIBDIR=lib \
-DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \
-DCMAKE_CXX_FLAGS=${CXXFLAGS:-} \
-DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \
-DARROW_VERBOSE_THIRDPARTY_BUILD=${ARROW_VERBOSE_THIRDPARTY_BUILD:-OFF} \
-DBUILD_WARNING_LEVEL=${DARROW_BUILD_WARNING_LEVEL:-CHECKIN} \
-DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS:-ON} \
-DARROW_BUILD_BENCHMARKS=${ARROW_BUILD_BENCHMARKS:-ON} \
-DARROW_BUILD_UTILITIES=${ARROW_BUILD_UTILITIES:-ON} \
-DARROW_BUILD_EXAMPLES=${ARROW_BUILD_EXAMPLES:-ON} \
-DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED:-ON} \
-DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC:-ON} \
-DARROW_NO_DEPRECATED_API=${ARROW_NO_DEPRECATED_API:-OFF} \
-DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \
-DARROW_USE_UBSAN=${ARROW_USE_UBSAN:-OFF} \
-DARROW_FLIGHT=${ARROW_FLIGHT:-ON} \
-DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \
-DARROW_MIMALLOC=${ARROW_MIMALLOC:-ON} \
-DARROW_ORC=${ARROW_ORC:-ON} \
-DARROW_PLASMA=${ARROW_PLASMA:-ON} \
-DARROW_PARQUET=${ARROW_PARQUET:-ON} \
-DARROW_HDFS=${ARROW_HDFS:-OFF} \
-DARROW_PYTHON=${ARROW_PYTHON:-OFF} \
-DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \
-DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \
-DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-ON} \
-DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY:-ON} \
-DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-ON} \
-DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-ON} \
-DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-ON} \
-DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-ON} \
-DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \
-DARROW_EXTRA_ERROR_CONTEXT=ON \
-DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \
-DPARQUET_REQUIRE_ENCRYPTION=${ARROW_WITH_OPENSSL:-ON} \
-DBOOST_SOURCE=${ARROW_BOOST_SOURCE:-AUTO} \
-Duriparser_SOURCE=AUTO \
-DARROW_FUZZING=${ARROW_FUZZING:-OFF} \
-DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \
${CMAKE_ARGS} \
${source_dir}
ninja
ninja install
popd
ci/docker_build_python.sh
-52
View File
@@ -1,52 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -e
source_dir=${1:-/arrow/python}
build_dir=${2:-/build/python}
export PYARROW_CXXFLAGS=$CXXFLAGS
export PYARROW_CMAKE_GENERATOR=Ninja
export PYARROW_BUILD_TYPE=${PYARROW_BUILD_TYPE:-debug}
# Feature flags
export SETUPTOOLS_SCM_VERSION_WRITE_TO_PREFIX=$build_dir
export PYARROW_WITH_ORC=${PYARROW_WITH_ORC:-1}
export PYARROW_WITH_PARQUET=${PYARROW_WITH_PARQUET:-1}
export PYARROW_WITH_PLASMA=${PYARROW_WITH_PLASMA:-1}
# Build pyarrow
pushd ${source_dir}
# hacky again, setuptools_scm writes _generated_version.py before pyarrow
# directory is created by setuptools
rm -rf ${build_dir}
mkdir -p $build_dir/pyarrow
relative_build_dir=$(realpath --relative-to=. $build_dir)
# this is a nightmare, but prevents mutating the source directory
# which is bind mounted as readonly
python setup.py build_ext --build-temp $relative_build_dir \
--build-lib $relative_build_dir \
build_py --build-lib $relative_build_dir \
egg_info --egg-base $relative_build_dir \
install_lib --build-dir $relative_build_dir \
install --single-version-externally-managed \
--record $relative_build_dir/record.txt
popd
ci/docker_integration.sh
-120
View File
@@ -1,120 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -eu
BIN_DIR=/bin
CPP_DIR=/build/cpp
JAVA_DIR=/build/java
JS_DIR=/build/js
# Copy various stuff into our Docker build directory
pushd /arrow
rsync -r \
LICENSE.txt NOTICE.txt \
header java js format integration testing /build
popd
mkdir -p $BIN_DIR
mkdir -p $CPP_DIR
# Install archery tool, require writable -e install
cp -r /arrow/dev/archery /archery
pip install -e /archery
# archery requires NumPy
pip install numpy
# Archery integration test environment variables
export ARROW_ROOT=/build
export ARROW_TEST_DATA=/arrow/testing/data
# Make a minimal C++ build for running integration tests
function build_cpp() {
pushd $CPP_DIR
cmake /arrow/cpp \
-DCMAKE_BUILD_TYPE=debug \
-DBOOST_SOURCE=SYSTEM \
-DARROW_BUILD_INTEGRATION=ON \
-DARROW_FLIGHT=ON \
-DARROW_COMPUTE=OFF \
-DARROW_DATASET=OFF \
-DARROW_FILESYSTEM=OFF \
-DARROW_HDFS=OFF \
-DARROW_JEMALLOC=OFF \
-DARROW_JSON=OFF \
-DARROW_USE_GLOG=OFF
make -j4 arrow-integration
export ARROW_CPP_EXE_PATH=$CPP_DIR/debug
}
function build_java() {
export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
# /arrow/java is read-only
pushd /arrow
popd
pushd $JAVA_DIR
mvn package -B -DskipTests -Drat.skip=true
popd
}
# Build Go
function build_go() {
pushd /arrow/go/arrow
export GO111MODULE=on
export GOBIN=$BIN_DIR
which go
go version
go env
go get -v ./...
popd
}
# Build JS
function build_js() {
source $NVM_DIR/nvm.sh
nvm install $NODE_VERSION
nvm alias default $NODE_VERSION
nvm use default
pushd $JS_DIR
npm install
npm run build -- -t apache-arrow
popd
}
# Run integration tests
GOLD_14_1=$ARROW_TEST_DATA/arrow-ipc/integration/0.14.1
build_cpp
build_go
build_js
build_java
archery integration --with-all --run-flight
ci/docker_java_test_all.sh
-34
View File
@@ -1,34 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -e
SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export ARROW_TEST_DATA=/arrow/testing/data
export ARROW_JAVA_RUN_TESTS=1
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
export ARROW_JAVADOC=1
bash $SOURCE_DIR/docker_build_java.sh
export ARROW_JAVA_SHADE_FLATBUFS=1
export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
export ARROW_JAVADOC=0
bash $SOURCE_DIR/docker_build_java.sh
integration/spark/ARROW-6429.patch → ci/etc/ARROW-6429.patch
View File
File renamed without changes.
integration/hdfs/hdfs-site.xml → ci/etc/hdfs-site.xml
View File
File renamed without changes.
ci/go-build-main.bat
-26
View File
@@ -1,26 +0,0 @@
@rem Licensed to the Apache Software Foundation (ASF) under one
@rem or more contributor license agreements. See the NOTICE file
@rem distributed with this work for additional information
@rem regarding copyright ownership. The ASF licenses this file
@rem to you under the Apache License, Version 2.0 (the
@rem "License"); you may not use this file except in compliance
@rem with the License. You may obtain a copy of the License at
@rem
@rem http://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing,
@rem software distributed under the License is distributed on an
@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@rem KIND, either express or implied. See the License for the
@rem specific language governing permissions and limitations
@rem under the License.
@rem The "main" Go build script for Windows CI
pushd go\arrow
%GOROOT%\bin\go version
%GOROOT%\bin\go get -v ./... || exit /B
%GOROOT%\bin\go test ./... || exit /B
popd
ci/rust-build-main.bat
-60
View File
@@ -1,60 +0,0 @@
@rem Licensed to the Apache Software Foundation (ASF) under one
@rem or more contributor license agreements. See the NOTICE file
@rem distributed with this work for additional information
@rem regarding copyright ownership. The ASF licenses this file
@rem to you under the Apache License, Version 2.0 (the
@rem "License"); you may not use this file except in compliance
@rem with the License. You may obtain a copy of the License at
@rem
@rem http://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing,
@rem software distributed under the License is distributed on an
@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@rem KIND, either express or implied. See the License for the
@rem specific language governing permissions and limitations
@rem under the License.
@rem The "main" Rust build script for Windows CI
@rem Retrieve git submodules, configure env var for Parquet unit tests
git submodule update --init || exit /B
set ARROW_TEST_DATA=%CD%\testing\data
set PARQUET_TEST_DATA=%CD%\cpp\submodules\parquet-testing\data
pushd rust
@echo ===================================
@echo Build with nightly toolchain
@echo ===================================
rustup default nightly
rustup show
cargo build --target %TARGET% --all-targets --release || exit /B
@echo
@echo Build with no default features
@echo ------------------------------
pushd arrow
cargo build --target %TARGET% --all-targets --no-default-features || exit /B
popd
@echo
@echo Test (release)
@echo --------------
cargo test --target %TARGET% --release || exit /B
@echo
@echo Run Arrow examples (release)
@echo ---------------------
pushd arrow
cargo run --example builders --target %TARGET% --release || exit /B
cargo run --example dynamic_types --target %TARGET% --release || exit /B
cargo run --example read_csv --target %TARGET% --release || exit /B
cargo run --example read_csv_infer_schema --target %TARGET% --release || exit /B
popd
@echo
@echo Run DataFusion examples (release)
@echo ---------------------
pushd datafusion
cargo run --example csv_sql --target %TARGET% --release || exit /B
cargo run --example parquet_sql --target %TARGET% --release || exit /B
popd
ci/docker_build_c_glib.sh → ci/scripts/c_glib_build.sh
+13 -10
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,22 +17,25 @@
# specific language governing permissions and limitations
# under the License.
set -e
set -ex
export ARROW_C_GLIB_HOME=$CONDA_PREFIX
source_dir=${1}/c_glib
build_dir=${2:-${source_dir}/build}
with_docs=${3:-false}
export CFLAGS="-DARROW_NO_DEPRECATED_API"
export CXXFLAGS="-DARROW_NO_DEPRECATED_API"
mkdir -p /build/c_glib
mkdir -p ${build_dir}
# Build with Meson
meson --prefix=$ARROW_C_GLIB_HOME \
meson --prefix=$ARROW_HOME \
--libdir=lib \
/build/c_glib \
/arrow/c_glib
-Dgtk_doc=${with_docs} \
${build_dir} \
${source_dir}
pushd /build/c_glib
ninja
ninja install
pushd ${build_dir}
ninja
ninja install
popd
ci/travis_script_cpp.sh → ci/scripts/c_glib_test.sh
+20 -16
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,25 +17,29 @@
# specific language governing permissions and limitations
# under the License.
set -e
set -ex
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
source_dir=${1}/c_glib
build_dir=${2:-${source_dir}/build}
pushd $CPP_BUILD_DIR
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig
export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0
export PKG_CONFIG=/usr/bin/pkg-config
if [ $TRAVIS_OS_NAME == "osx" ]; then
# TODO: This does not seem to terminate
CTEST_ARGS="-E arrow-flight-test"
fi
pushd ${source_dir}
PATH=$ARROW_BUILD_TYPE:$PATH ctest -j2 --output-on-failure -L unittest ${CTEST_ARGS}
ruby test/run-test.rb
popd
# Capture C++ coverage info (we wipe the build dir in travis_script_python.sh)
if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then
pushd $TRAVIS_BUILD_DIR
lcov --directory . --capture --no-external --output-file $ARROW_CPP_COVERAGE_FILE \
2>&1 | grep -v "ignoring data for external file"
if [[ "$(uname -s)" == "Linux" ]]; then
# TODO(kszucs): on osx it fails to load 'lgi.corelgilua51' despite that lgi
# was installed by luarocks
pushd example/lua
lua write-batch.lua
lua read-batch.lua
lua write-stream.lua
lua read-stream.lua
popd
fi
popd
ci/scripts/cpp_build.sh
Executable
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -ex
source_dir=${1}/cpp
build_dir=${2:-${source_dir}/build}
with_docs=${3:-false}
# TODO(kszucs): consider to move these to CMake
if [ ! -z "${CONDA_PREFIX}" ]; then
export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_AR=${AR} -DCMAKE_RANLIB=${RANLIB}"
export ARROW_GANDIVA_PC_CXX_FLAGS=$(echo | ${CXX} -E -Wp,-v -xc++ - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';')
elif [ -x "$(command -v xcrun)" ]; then
export ARROW_GANDIVA_PC_CXX_FLAGS="-isysroot;$(xcrun --show-sdk-path)"
fi
# export CCACHE_COMPRESS=1
# export CCACHE_COMPRESSLEVEL=5
# export CCACHE_COMPILERCHECK=content
# export CCACHE_DIR=/build/ccache
# export PATH=/usr/lib/ccache/:$PATH
mkdir -p ${build_dir}
pushd ${build_dir}
cmake -G "${CMAKE_GENERATOR:-Ninja}" \
-DARROW_BOOST_USE_SHARED=${ARROW_BOOST_USE_SHARED:-ON} \
-DARROW_BUILD_BENCHMARKS=${ARROW_BUILD_BENCHMARKS:-OFF} \
-DARROW_BUILD_EXAMPLES=${ARROW_BUILD_EXAMPLES:-OFF} \
-DARROW_BUILD_INTEGRATION=${ARROW_BUILD_INTEGRATION:-OFF} \
-DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED:-ON} \
-DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC:-ON} \
-DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS:-OFF} \
-DARROW_BUILD_UTILITIES=${ARROW_BUILD_UTILITIES:-ON} \
-DARROW_COMPUTE=${ARROW_COMPUTE:-ON} \
-DARROW_CUDA=${ARROW_CUDA:-OFF} \
-DARROW_CXXFLAGS=${ARROW_CXXFLAGS:-} \
-DARROW_DATASET=${ARROW_DATASET:-ON} \
-DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \
-DARROW_EXTRA_ERROR_CONTEXT=${ARROW_EXTRA_ERROR_CONTEXT:-OFF} \
-DARROW_FILESYSTEM=${ARROW_FILESYSTEM:-ON} \
-DARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \
-DARROW_FUZZING=${ARROW_FUZZING:-OFF} \
-DARROW_FUZZING=${ARROW_FUZZING:-OFF} \
-DARROW_JNI=${ARROW_JNI:-OFF} \
-DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \
-DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \
-DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \
-DARROW_HDFS=${ARROW_HDFS:-ON} \
-DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \
-DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \
-DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \
-DARROW_LARGE_MEMORY_TESTS=${ARROW_LARGE_MEMORY_TESTS:-OFF} \
-DARROW_MIMALLOC=${ARROW_MIMALLOC:-OFF} \
-DARROW_NO_DEPRECATED_API=${ARROW_NO_DEPRECATED_API:-OFF} \
-DARROW_ORC=${ARROW_ORC:-OFF} \
-DARROW_PARQUET=${ARROW_PARQUET:-OFF} \
-DARROW_PLASMA=${ARROW_PLASMA:-OFF} \
-DARROW_PYTHON=${ARROW_PYTHON:-OFF} \
-DARROW_S3=${ARROW_S3:-OFF} \
-DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \
-DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \
-DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \
-DARROW_USE_GLOG=${ARROW_USE_GLOG:-ON} \
-DARROW_USE_STATIC_CRT=${ARROW_USE_STATIC_CRT:-OFF} \
-DARROW_USE_UBSAN=${ARROW_USE_UBSAN:-OFF} \
-DARROW_VERBOSE_THIRDPARTY_BUILD=${ARROW_VERBOSE_THIRDPARTY_BUILD:-OFF} \
-DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-OFF} \
-DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-OFF} \
-DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-OFF} \
-DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY:-OFF} \
-DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \
-DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \
-Dbenchmark_SOURCE=${benchmark_SOURCE:-AUTO} \
-DBOOST_SOURCE=${ARROW_BOOST_SOURCE:-AUTO} \
-DBOOST_SOURCE=${BOOST_SOURCE:-AUTO} \
-DBrotli_SOURCE=${Brotli_SOURCE:-AUTO} \
-DBUILD_WARNING_LEVEL=${DARROW_BUILD_WARNING_LEVEL:-CHECKIN} \
-Dc-ares_SOURCE=${cares_SOURCE:-AUTO} \
-DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \
-DCMAKE_CXX_FLAGS=${CXXFLAGS:-} \
-DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \
-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \
-Ddouble-conversion_SOURCE=${uriparser_SOURCE:-AUTO} \
-Dgflags_SOURCE=${gflags_SOURCE:-AUTO} \
-DgRPC_SOURCE=${gRPC_SOURCE:-AUTO} \
-DGTest_SOURCE=${GTest_SOURCE:-AUTO} \
-DLz4_SOURCE=${Lz4_SOURCE:-AUTO} \
-DORC_SOURCE=${ORC_SOURCE:-AUTO} \
-DPARQUET_REQUIRE_ENCRYPTION=${ARROW_WITH_OPENSSL:-ON} \
-DProtobuf_SOURCE=${Protobuf_SOURCE:-AUTO} \
-DRapidJSON_SOURCE=${RapidJSON_SOURCE:-AUTO} \
-DRE2_SOURCE=${RE2_SOURCE:-AUTO} \
-DThrift_SOURCE=${Thrift_SOURCE:-AUTO} \
-Duriparser_SOURCE=${uriparser_SOURCE:-AUTO} \
-DZSTD_SOURCE=${ZSTD_SOURCE:-AUTO} \
${CMAKE_ARGS} \
${source_dir}
cmake --build . --target install
popd
if [ "${with_docs}" == "true" ]; then
pushd ${source_dir}/apidoc
doxygen
popd
fi
ci/travis_script_java.sh → ci/scripts/cpp_test.sh
+11 -11
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,18 +17,18 @@
# specific language governing permissions and limitations
# under the License.
set -e
set -ex
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
arrow_dir=${1}
source_dir=${1}/cpp
build_dir=${2:-${source_dir}/build}
JAVA_DIR=${TRAVIS_BUILD_DIR}/java
pushd $JAVA_DIR
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${source_dir}/submodules/parquet-testing/data
export LD_LIBRARY_PATH=${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}:${LD_LIBRARY_PATH}
if [ "$ARROW_TRAVIS_JAVA_BUILD_ONLY" == "1" ]; then
# Save time and make build less verbose by skipping tests and style checks
$TRAVIS_MVN -DskipTests=true -Dcheckstyle.skip=true -B install
else
$TRAVIS_MVN -B install
fi
pushd ${build_dir}
ctest --output-on-failure -j$(nproc)
popd
ci/docker_build_and_test_python.sh → ci/scripts/csharp_build.sh
+7 -4
View File
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,8 +17,10 @@
# specific language governing permissions and limitations
# under the License.
set -e
set -ex
/arrow/ci/docker_build_cpp.sh
/arrow/ci/docker_build_python.sh
pytest -v --pyargs pyarrow
source_dir=${1}/csharp
pushd ${source_dir}
dotnet build
popd
dev/Dockerfile.context → ci/scripts/csharp_test.sh
Regular → Executable
+8 -3
View File
@@ -1,3 +1,5 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,7 +17,10 @@
# specific language governing permissions and limitations
# under the License.
FROM ubuntu:latest
set -ex
COPY . /context
CMD du -sh /context/*
source_dir=${1}/csharp
pushd ${source_dir}
dotnet test
popd
ci/docker_build_sphinx.sh → ci/scripts/docs_build.sh
+21 -7
View File
@@ -18,12 +18,26 @@
set -ex
pushd /arrow/cpp/apidoc
doxygen
popd
arrow_dir=${1}
build_dir=${2}
sphinx-build -b html /arrow/docs/source /arrow/docs/_build/html
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig:${PKG_CONFIG_PATH}
export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0
export CFLAGS="-DARROW_NO_DEPRECATED_API"
export CXXFLAGS="-DARROW_NO_DEPRECATED_API"
git clone -b asf-site https://github.com/apache/arrow-site /arrow/site
mkdir -p /arrow/site/docs/latest
rsync -r /arrow/docs/_build/html/ /arrow/site/docs/latest/
# Prose and Python
sphinx-build -b html ${arrow_dir}/docs/source ${build_dir}/sphinx
# C++
rsync -a --delete ${arrow_dir}/cpp/apidoc/* ${build_dir}/cpp
# C GLib
rsync -a --delete ${ARROW_HOME}/share/gtk-doc/html/* ${build_dir}/c_glib
# Java
rsync -a --delete ${arrow_dir}/java/target/site/apidocs ${build_dir}/java
# Javascript
rsync -a --delete ${arrow_dir}/js/doc ${build_dir}/js
ci/travis_script_plasma_java_client.sh → ci/scripts/fuzzit_build.sh
+19 -14
View File
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,25 +16,31 @@
# specific language governing permissions and limitations
# under the License.
set -e
set -euxo pipefail
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
export TARGET_ID=apache-arrow/arrow-ipc-fuzzing
source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
source_dir=${1}/cpp
build_dir=${2:-${source_dir}/build}
conda activate $CPP_TOOLCHAIN
pushd ${build_dir}/relwithdebinfo
PLASMA_JAVA_DIR=${TRAVIS_BUILD_DIR}/java/plasma
mkdir -p out
pushd $PLASMA_JAVA_DIR
cp arrow-ipc-fuzzing-test out/fuzzer
ldd arrow-ipc-fuzzing-test | grep "=> /" | awk '{print $3}' | xargs -I '{}' cp -v '{}' out/
$TRAVIS_MVN clean install
pushd out
tar -czvf fuzzer.tar.gz *
stat fuzzer.tar.gz
popd
export LD_LIBRARY_PATH=${ARROW_CPP_INSTALL}/lib:$LD_LIBRARY_PATH
export PLASMA_STORE=${ARROW_CPP_INSTALL}/bin/plasma_store_server
ldd $PLASMA_STORE
$TRAVIS_JAVA -cp target/test-classes:target/classes -Djava.library.path=${TRAVIS_BUILD_DIR}/cpp-build/debug/ org.apache.arrow.plasma.PlasmaClientTest
fuzzit create job \
--type $FUZZIT_JOB_TYPE \
--host $FUZZIT_HOST \
--revision $CI_ARROW_SHA \
--branch $CI_ARROW_BRANCH \
$TARGET_ID \
out/fuzzer.tar.gz
popd
integration/pandas/Dockerfile → ci/scripts/go_build.sh
Regular → Executable
+11 -3
View File
@@ -1,3 +1,5 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,7 +17,13 @@
# specific language governing permissions and limitations
# under the License.
FROM arrowdev/arrow-python-3.6:latest
set -ex
# test against pandas' master
RUN pip install git+https://github.com/pandas-dev/pandas.git
source_dir=${1}/go
pushd ${source_dir}/arrow
go get -d -t -v ./...
go install -v ./...
popd
ci/scripts/go_test.sh
Executable
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -ex
source_dir=${1}/go
pushd ${source_dir}/arrow
for d in $(go list ./... | grep -v vendor); do
go test $d
done
popd
ci/scripts/install_conda.sh
Executable
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -e
declare -A archs
archs=([amd64]=x86_64
[arm32v7]=armv7l
[ppc64le]=ppc64le
[i386]=x86)
declare -A platforms
platforms=([windows]=Windows
[macos]=MacOSX
[linux]=Linux)
if [ "$#" -ne 4 ]; then
echo "Usage: $0 <architecture> <platform> <version> <prefix>"
exit 1
elif [[ -z ${archs[$1]} ]]; then
echo "Unexpected architecture: ${1}"
exit 1
elif [[ -z ${platforms[$2]} ]]; then
echo "Unexpected platform: ${2}"
exit 1
fi
arch=${archs[$1]}
platform=${platforms[$2]}
version=$3
prefix=$4
echo "Downloading Miniconda installer..."
wget -nv https://repo.continuum.io/miniconda/Miniconda3-${version}-${platform}-${arch}.sh -O /tmp/miniconda.sh
bash /tmp/miniconda.sh -b -p ${prefix}
rm /tmp/miniconda.sh
ln -s ${prefix}/etc/profile.d/conda.sh /etc/profile.d/conda.sh
echo "conda activate base" >> ~/.profile
# Configure
source /etc/profile.d/conda.sh
conda config --add channels conda-forge
conda config --set channel_priority strict
conda config --set show_channel_urls True
conda config --set remote_connect_timeout_secs 12
# Update and clean
conda update --all -y
conda clean --all -y
ci/docker_install_iwyu.sh → ci/scripts/install_iwyu.sh
+15 -11
View File
@@ -18,22 +18,26 @@
set -eu
: ${IWYU_REPO:="https://github.com/include-what-you-use/include-what-you-use.git"}
: ${IWYU_BRANCH:="clang_7.0"}
: ${IWYU_SRC:="/tmp/iwyu"}
: ${IWYU_HOME:="/opt/iwyu"}
source_dir=${1:-/tmp/iwyu}
install_prefix=${2:-/usr/local}
llvm_major=${3:-7}
git clone "${IWYU_REPO}" "${IWYU_SRC}"
git -C "${IWYU_SRC}" checkout ${IWYU_BRANCH}
git clone --single-branch --branch "clang_${llvm_major}.0" \
https://github.com/include-what-you-use/include-what-you-use.git ${source_dir}
mkdir -p "${IWYU_HOME}"
pushd "${IWYU_HOME}"
mkdir -p ${source_dir}/build
pushd ${source_dir}/build
# Build IWYU for current Clang
export CC=clang-7
export CXX=clang++-7
export CC=clang-${llvm_major}
export CXX=clang++-${llvm_major}
cmake -DCMAKE_PREFIX_PATH=/usr/lib/llvm-7 "${IWYU_SRC}"
cmake -DCMAKE_PREFIX_PATH=/usr/lib/llvm-${llvm_major} \
-DCMAKE_INSTALL_PREFIX=${install_prefix} \
${source_dir}
make -j4
make install
popd
rm -rf ${source_dir}
ci/scripts/install_minio.sh
Executable
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -e
declare -A archs
archs=([amd64]=amd64)
declare -A platforms
platforms=([macos]=darwin
[linux]=linux)
arch=${archs[$1]}
platform=${platforms[$2]}
version=$3
prefix=$4
if [ "$#" -ne 4 ]; then
echo "Usage: $0 <architecture> <platform> <version> <prefix>"
exit 1
elif [[ -z ${archs[$1]} ]]; then
echo "Unexpected architecture: ${1}"
exit 1
elif [[ -z ${platforms[$2]} ]]; then
echo "Unexpected platform: ${2}"
exit 1
elif [[ ${version} != "latest" ]]; then
echo "Cannot fetch specific versions of minio, only latest is supported."
exit 1
fi
wget -nv -P ${prefix}/bin https://dl.min.io/server/minio/release/linux-${arch}/minio
chmod +x ${prefix}/bin/minio
integration/dask/runtest.sh → ci/scripts/integration_dask.sh
View File
File renamed without changes.
integration/hdfs/runtest.sh → ci/scripts/integration_hdfs.sh
+6 -4
View File
@@ -17,21 +17,23 @@
# specific language governing permissions and limitations
# under the License.
# exit on any error
set -e
source_dir=${1}/cpp
build_dir=${2:-${source_dir}/build}
export CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath --glob`
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export LIBHDFS3_CONF=$HADOOP_CONF_DIR/hdfs-site.xml
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native/
# execute cpp tests
pushd /build/cpp
debug/arrow-io-hdfs-test
pushd ${build_dir}
debug/arrow-io-hdfs-test
popd
# cannot use --pyargs with custom arguments like --hdfs or --only-hdfs, because
# pytest ignores them, see https://github.com/pytest-dev/pytest/issues/3517
export PYARROW_TEST_ONLY_HDFS=ON
export PYARROW_TEST_HDFS=ON
pytest -v --pyargs pyarrow.tests.test_hdfs
dev/run_site.sh → ci/scripts/integration_hiveserver2.sh
+16 -3
View File
@@ -16,6 +16,19 @@
# limitations under the License.
#
# Pass the service name to run_docker_compse.sh
# Which validates environment and runs the service
exec "$(dirname ${BASH_SOURCE})"/run_docker_compose.sh run_site
set -e
arrow_dir=${1}
source_dir=${1}/cpp
build_dir=${2:-${source_dir}/build}
${arrow_dir}/ci/scripts/wait_for_it.sh impala:21050 -t 300 -s -- echo "impala is up"
pushd ${build_dir}
# ninja hiveserver2-test
debug/hiveserver2-test
popd
integration/spark/runtest.sh → ci/scripts/integration_spark.sh
+14 -14
View File
@@ -16,38 +16,38 @@
# limitations under the License.
# exit on any error
set -e
set -eu
# rsynced source directory to build java libs
arrow_src=/build/java/arrow
source_dir=${1}
spark_dir=${2}
pushd $arrow_src/java
ARROW_VERSION=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | sed -n -e '/^\[.*\]/ !{ /^[0-9]/ { p; q } }'`
pushd ${source_dir}/java
arrow_version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | sed -n -e '/^\[.*\]/ !{ /^[0-9]/ { p; q } }'`
popd
export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=warn"
export MAVEN_OPTS="${MAVEN_OPTS} -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
# build Spark with Arrow
pushd /spark
pushd ${spark_dir}
# update Spark pom with the Arrow version just installed and build Spark, need package phase for pyspark
echo "Building Spark with Arrow $ARROW_VERSION"
mvn versions:set-property -Dproperty=arrow.version -DnewVersion=$ARROW_VERSION
echo "Building Spark with Arrow ${arrow_version}"
mvn versions:set-property -Dproperty=arrow.version -DnewVersion=${arrow_version}
build/mvn -B -DskipTests package -pl sql/core -pl assembly -am
SPARK_SCALA_TESTS=(
spark_scala_tests=(
"org.apache.spark.sql.execution.arrow"
"org.apache.spark.sql.execution.vectorized.ColumnarBatchSuite"
"org.apache.spark.sql.execution.vectorized.ArrowColumnVectorSuite")
(echo "Testing Spark:"; IFS=$'\n'; echo "${SPARK_SCALA_TESTS[*]}")
(echo "Testing Spark:"; IFS=$'\n'; echo "${spark_scala_tests[*]}")
# TODO: should be able to only build spark-sql tests with adding "-pl sql/core" but not currently working
build/mvn -B -Dtest=none -DwildcardSuites=$(IFS=,; echo "${SPARK_SCALA_TESTS[*]}") test
build/mvn -B -Dtest=none -DwildcardSuites=$(IFS=,; echo "${spark_scala_tests[*]}") test
# Run pyarrow related Python tests only
SPARK_PYTHON_TESTS=(
spark_python_tests=(
"pyspark.sql.tests.test_arrow"
"pyspark.sql.tests.test_pandas_udf"
"pyspark.sql.tests.test_pandas_udf_scalar"
@@ -55,6 +55,6 @@ pushd /spark
"pyspark.sql.tests.test_pandas_udf_grouped_map"
"pyspark.sql.tests.test_pandas_udf_window")
(echo "Testing PySpark:"; IFS=$'\n'; echo "${SPARK_PYTHON_TESTS[*]}")
python/run-tests --testnames "$(IFS=,; echo "${SPARK_PYTHON_TESTS[*]}")" --python-executables python
(echo "Testing PySpark:"; IFS=$'\n'; echo "${spark_python_tests[*]}")
python/run-tests --testnames "$(IFS=,; echo "${spark_python_tests[*]}")" --python-executables python
popd
integration/turbodbc/runtest.sh → ci/scripts/integration_turbodbc.sh
+14 -13
View File
@@ -19,27 +19,28 @@
set -e
source_dir=${1}
build_dir=${2:-${source_dir}/build}
# check that optional pyarrow modules are available
# because pytest would just skip the pyarrow tests
python -c "import pyarrow.orc"
python -c "import pyarrow.parquet"
pushd /tmp
git clone https://github.com/blue-yonder/turbodbc.git
pushd turbodbc
git submodule update --init --recursive
mkdir -p ${build_dir}
pushd ${build_dir}
service postgresql start
sudo -u postgres psql -U postgres -c 'CREATE DATABASE test_db;'
sudo -u postgres psql -U postgres -c 'ALTER USER postgres WITH PASSWORD '\''password'\'';'
export ODBCSYSINI="$(pwd)/travis/odbc/"
mkdir build
pushd build
cmake -DCMAKE_INSTALL_PREFIX=./dist -DPYTHON_EXECUTABLE=`which python` -GNinja ..
cmake -DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \
-DCMAKE_CXX_FLAGS=${CXXFLAGS} \
-GNinja \
${source_dir}
ninja install
# TODO(ARROW-5074)
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/conda/lib"
export LD_LIBRARY_PATH="${ARROW_HOME}/lib:${LD_LIBRARY_PATH}"
export ODBCSYSINI="${source_dir}/travis/odbc/"
service postgresql start
ctest --output-on-failure
popd
ci/scripts/java_build.sh
Executable
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -ex
arrow_dir=${1}
source_dir=${1}/java
cpp_build_dir=${2:-${arrow_dir}/cpp/build/debug}
with_docs=${3:-false}
mvn="mvn -B -DskipTests -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
pushd ${source_dir}
${mvn} install
if [ "${ARROW_JAVA_SHADE_FLATBUFFERS}" == "ON" ]; then
${mvn} -Pshade-flatbuffers install
fi
if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then
${mvn} -Darrow.cpp.build.dir=${cpp_build_dir} -Parrow-jni install
fi
if [ "${ARROW_PLASMA}" = "ON" ]; then
pushd ${source_dir}/plasma
${mvn} clean install
popd
fi
if [ "${with_docs}" == "true" ]; then
${mvn} -Dcheckstyle.skip=true install site
fi
popd
ci/docker_build_java.sh → ci/scripts/java_test.sh
+18 -22
View File
@@ -16,36 +16,32 @@
# specific language governing permissions and limitations
# under the License.
set -e
set -ex
export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
arrow_dir=${1}
source_dir=${1}/java
cpp_build_dir=${2:-${arrow_dir}/cpp/build/debug}
# /arrow/java is read-only
mkdir -p /build/java
# for jni and plasma tests
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
export PLASMA_STORE=${ARROW_HOME}/bin/plasma_store_server
arrow_src=/build/java/arrow
mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
# Remove any pre-existing artifacts
rm -rf $arrow_src
pushd ${source_dir}
pushd /arrow
rsync -a header java format integration testing $arrow_src
popd
${mvn} test
JAVA_ARGS=
if [ "$ARROW_JAVA_RUN_TESTS" != "1" ]; then
JAVA_ARGS=-DskipTests
if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then
${mvn} test -Parrow-jni -pl gandiva -Darrow.cpp.build.dir=${cpp_build_dir}
fi
if [ "$ARROW_JAVA_SHADE_FLATBUFS" == "1" ]; then
SHADE_FLATBUFFERS=-Pshade-flatbuffers
if [ "${ARROW_PLASMA}" = "ON" ]; then
pushd ${source_dir}/plasma
java -cp target/test-classes:target/classes \
-Djava.library.path=${cpp_build_dir} \
org.apache.arrow.plasma.PlasmaClientTest
popd
fi
pushd $arrow_src/java
mvn -B $JAVA_ARGS -Drat.skip=true install $SHADE_FLATBUFFERS
if [ "$ARROW_JAVADOC" == "1" ]; then
export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=warn"
mvn -B site
fi
popd
ci/travis_script_js.sh → ci/scripts/js_build.sh
+11 -6
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -19,14 +19,19 @@
set -ex
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
source_dir=${1}/js
with_docs=${2:-false}
pushd $ARROW_JS_DIR
pushd ${source_dir}
npm install
# TODO(kszucs): linting should be moved to archery
npm run lint:ci
npm run build
npm test
npm run test:coverage
bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports"
if [ "${with_docs}" == "true" ]; then
npm install typedoc
npm run doc
fi
popd
csharp/build/docker/Dockerfile → ci/scripts/js_test.sh
Regular → Executable
+10 -4
View File
@@ -1,3 +1,5 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,9 +17,13 @@
# specific language governing permissions and limitations
# under the License.
FROM mcr.microsoft.com/dotnet/core/sdk:2.1
set -ex
COPY csharp /arrow/csharp
WORKDIR /arrow/csharp
source_dir=${1}/js
CMD ["dotnet", "build"]
pushd ${source_dir}
npm run lint
npm run test
popd
ci/docker_install_conda.sh → ci/scripts/python_benchmark.sh
+20 -20
View File
@@ -17,26 +17,26 @@
# specific language governing permissions and limitations
# under the License.
# Exit on any error
set -e
# TODO(kszucs): port it
wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/miniconda.sh
bash /tmp/miniconda.sh -b -q -p ${CONDA_PREFIX:=/opt/conda}
rm /tmp/miniconda.sh
# Check the ASV benchmarking setup.
# Unfortunately this won't ensure that all benchmarks succeed
# (see https://github.com/airspeed-velocity/asv/issues/449)
source deactivate
conda create -y -q -n pyarrow_asv python=$PYTHON_VERSION
conda activate pyarrow_asv
pip install -q git+https://github.com/pitrou/asv.git@customize_commands
ln -s ${CONDA_PREFIX}/etc/profile.d/conda.sh /etc/profile.d/conda.sh
echo ". ${CONDA_PREFIX}/etc/profile.d/conda.sh" >> ~/.bashrc
echo "conda activate base" >> ~/.bashrc
export PYARROW_WITH_PARQUET=1
export PYARROW_WITH_PLASMA=1
export PYARROW_WITH_ORC=0
export PYARROW_WITH_GANDIVA=0
# Configure conda
source $MINICONDA/etc/profile.d/conda.sh
conda config --set show_channel_urls True
# Help with SSL timeouts to S3
conda config --set remote_connect_timeout_secs 12
# Setup conda-forge
conda config --add channels conda-forge
# Update packages
conda update --all -q -y
pushd $ARROW_PYTHON_DIR
# Workaround for https://github.com/airspeed-velocity/asv/issues/631
git fetch --depth=100 origin master:master
# Generate machine information (mandatory)
asv machine --yes
# Run benchmarks on the changeset being tested
asv run --no-pull --show-stderr --quick HEAD^!
popd # $ARROW_PYTHON_DIR
ci/scripts/python_build.sh
Executable
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -ex
source_dir=${1}/python
build_dir=${2:-${source_dir}/build}
export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
export PYARROW_WITH_S3=${ARROW_S3:-OFF}
export PYARROW_WITH_ORC=${ARROW_ORC:-OFF}
export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF}
export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF}
export PYARROW_WITH_PLASMA=${ARROW_PLASMA:-OFF}
export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA:-OFF}
export PYARROW_WITH_PARQUET=${ARROW_PARQUET:-OFF}
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
pushd ${source_dir}
relative_build_dir=$(realpath --relative-to=. $build_dir)
# not nice, but prevents mutating the mounted the source directory for docker
python setup.py build_ext --build-temp $relative_build_dir \
--build-lib $relative_build_dir \
build_py --build-lib $relative_build_dir \
egg_info --egg-base $relative_build_dir \
install_lib --build-dir $relative_build_dir \
install --single-version-externally-managed \
--record $relative_build_dir/record.txt
popd
ci/travis_release_audit.sh → ci/scripts/python_test.sh
+9 -7
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,10 +17,12 @@
# specific language governing permissions and limitations
# under the License.
set -e
set -ex
# Check licenses according to Apache policy
git archive HEAD --prefix=apache-arrow-raw/ | tar xf -
cp -r -L apache-arrow{-raw,}
tar czf arrow-src.tar.gz apache-arrow
./dev/release/run-rat.sh arrow-src.tar.gz
arrow_dir=${1}
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
pytest -sv pyarrow
ci/docker_build_r.sh → ci/scripts/r_build.sh
+12 -14
View File
@@ -16,27 +16,25 @@
# specific language governing permissions and limitations
# under the License.
set -e
export ARROW_HOME=$CONDA_PREFIX
set -ex
: ${R_BIN:=R}
# Build arrow
pushd /arrow/r
source_dir=${1}/r
if [ "$R_CONDA" = "" ]; then
# Install R package dependencies
# NOTE: any changes here should also be done in docker_build_r_sanitizer.sh
# XXX ideally this sould be done from the Dockerfile, but I can't get it to work (AP)
${R_BIN} -e "remotes::install_deps(dependencies = TRUE)"
fi
# ARROW-6171: Because lz4 is installed in the base Ubuntu image, there's an
# issue of which library is loaded at runtime. R by default will override
# LD_LIBRARY_PATH at runtime by concatenating (in that order)
# R_LD_LIBRARY_PATH, R_JAVA_LD_LIBRARY_PATH and LD_LIBRARY_PATH. If
# R_LD_LIBRARY_PATH is not set, it'll default to a list of directories which
# contains /usr/lib/x86_64-linux-gnu.
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
export R_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
pushd ${source_dir}
make clean
${R_BIN} CMD build --keep-empty-dirs .
${R_BIN} CMD INSTALL $(ls | grep arrow_*.tar.gz)
export _R_CHECK_FORCE_SUGGESTS_=false
${R_BIN} CMD check $(ls | grep arrow_*.tar.gz) --as-cran --no-manual
popd
ci/docker_build_r_sanitizer.sh → ci/scripts/r_deps.sh
+7 -15
View File
@@ -16,25 +16,17 @@
# specific language governing permissions and limitations
# under the License.
set -e
set -ex
: ${R_BIN:=RDsan}
: ${R_BIN:=R}
# Build arrow
pushd /arrow/r
source_dir=${1}/r
pushd ${source_dir}
# Install R package dependencies
# NOTE: any changes here should also be done in docker_build_r.sh
${R_BIN} -e "install.packages(c('remotes', 'dplyr', 'glue'))"
${R_BIN} -e "remotes::install_deps(dependencies = TRUE)"
make clean
${R_BIN} CMD INSTALL --no-byte-compile .
pushd tests
export UBSAN_OPTIONS="print_stacktrace=1,suppressions=/arrow/r/tools/ubsan.supp"
${R_BIN} < testthat.R
popd
${R_BIN} -e "remotes::install_github('romainfrancois/decor')"
popd
ci/travis_before_script_js.sh → ci/scripts/r_sanitize.sh
+7 -7
View File
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,15 +16,16 @@
# specific language governing permissions and limitations
# under the License.
set -ex
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
: ${R_BIN:=RDsan}
pushd $ARROW_JS_DIR
source_dir=${1}/r
# ARROW(ARROW-2546): Update to npm>=5.7.1 to fight EINTEGRITY problems
npm install -g npm@latest
npm install
# Build arrow
pushd ${source_dir}/tests
export UBSAN_OPTIONS="print_stacktrace=1,suppressions=/arrow/r/tools/ubsan.supp"
${R_BIN} < testthat.R
popd
ci/travis_before_script_ruby.sh → ci/scripts/r_test.sh
+11 -10
View File
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,15 +16,17 @@
# specific language governing permissions and limitations
# under the License.
set -e
set -ex
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
: ${R_BIN:=R}
source_dir=${1}/r
pushd ${source_dir}
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
export R_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
${R_BIN} CMD check --no-manual --as-cran arrow_*.tar.gz
pushd $ARROW_RUBY_DIR/red-arrow
bundle install
popd
# TODO: GPU required
# pushd $ARROW_RUBY_DIR/red-arrow-gpu
# bundle install
# popd
ci/scripts/release_test.sh
Executable
+28
View File
@@ -0,0 +1,28 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -eux
arrow_dir=${1}
pushd ${arrow_dir}
dev/release/run-test.rb
popd
ci/scripts/ruby_test.sh
Executable
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -ex
source_dir=${1}/ruby
build_dir=${2:-${source_dir}/build}
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig
export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0
# TODO(kszucs): this should be moved into the dockerfile
bundle install --gemfile ${source_dir}/red-arrow/Gemfile
${source_dir}/red-arrow/test/run-test.rb
if [ ${ARROW_GANDIVA:-OFF} == "ON" ]; then
${source_dir}/red-gandiva/test/run-test.rb
fi
if [ ${ARROW_PARQUET:-OFF} == "ON" ]; then
${source_dir}/red-parquet/test/run-test.rb
fi
if [ ${ARROW_PLASMA:-OFF} == "ON" ]; then
${source_dir}/red-plasma/test/run-test.rb
fi
if [ ${ARROW_CUDA:-OFF} == "ON" ]; then
${source_dir}/red-arrow-cuda/test/run-test.rb
fi
ci/docker_build_rust.sh → ci/scripts/rust_build.sh
+14 -26
View File
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,49 +17,36 @@
# specific language governing permissions and limitations
# under the License.
set -e
set -ex
cd /arrow/rust
source_dir=${1}/rust
build_dir=${2:-${source_dir}/target}
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
export CARGO_TARGET_DIR=${build_dir}
# show activated toolchain
rustup show
# clean first
cargo clean
# ensure that the build directory exists
mkdir -p ${build_dir}
pushd ${source_dir}
# flatbuffer codegen
echo Generating IPC
echo "Generating IPC"
flatc --rust -o arrow/src/ipc/gen/ ../format/*.fbs
# work around some bugs in flatbuffers
find arrow/src/ipc/gen/ -name "*_generated.rs" -exec sed -i 's/type__type/type_type/g' {} \;
# raises on any formatting errors
echo "Running formatting checks ..."
cargo +stable fmt --all -- --check
echo "Formatting checks completed"
# build entire project
RUSTFLAGS="-D warnings" cargo build --all-targets
# run tests
cargo test
# make sure we can build Arrow sub-crate without default features
pushd arrow
cargo build --no-default-features
cargo build --all-targets --no-default-features
popd
# run Arrow examples
pushd arrow
cargo run --example builders
cargo run --example dynamic_types
cargo run --example read_csv
cargo run --example read_csv_infer_schema
popd
# run DataFusion examples
pushd datafusion
cargo run --example csv_sql
cargo run --example parquet_sql
popd
ci/travis_install_clang_tools.sh → ci/scripts/rust_test.sh
+28 -8
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,13 +17,33 @@
# specific language governing permissions and limitations
# under the License.
set -ex
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
arrow_dir=${1}
source_dir=${1}/rust
build_dir=${2:-${source_dir}/target}
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
sudo apt-add-repository -y \
"deb https://apt.llvm.org/$DISTRO_CODENAME/ llvm-toolchain-$DISTRO_CODENAME-$ARROW_LLVM_MAJOR_VERSION main"
sudo apt-get update -qq
sudo apt-get install -q clang-$ARROW_LLVM_MAJOR_VERSION clang-format-$ARROW_LLVM_MAJOR_VERSION clang-tidy-$ARROW_LLVM_MAJOR_VERSION
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
export CARGO_TARGET_DIR=${build_dir}
pushd ${source_dir}
# run unit tests
cargo test
# test arrow examples
pushd arrow
cargo run --example builders --release
cargo run --example dynamic_types --release
cargo run --example read_csv --release
cargo run --example read_csv_infer_schema --release
popd
# test datafusion examples
pushd datafusion
cargo run --example csv_sql --release
cargo run --example parquet_sql --release
popd
popd
dev/docker_common/wait-for-it.sh → ci/scripts/wait_for_it.sh
View File
File renamed without changes.
ci/travis_before_script_c_glib.sh
-81
View File
@@ -1,81 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -ex
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
if [ $TRAVIS_OS_NAME = "osx" ]; then
export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig
export XML_CATALOG_FILES=/usr/local/etc/xml/catalog
else
source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
conda create -n meson -y -q python=3.6
conda activate meson
pip install meson
sudo apt-get install -y -q \
autoconf-archive \
gtk-doc-tools \
libgirepository1.0-dev
conda install -q -y ninja
fi
if [ $TRAVIS_OS_NAME = "osx" ]; then
sudo env PKG_CONFIG_PATH=$PKG_CONFIG_PATH luarocks install lgi
else
sudo apt install -y -qq luarocks
sudo luarocks install lgi
fi
pushd $ARROW_C_GLIB_DIR
bundle install
export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib
# Build with GNU Autotools
./autogen.sh
CONFIGURE_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL_AUTOTOOLS"
CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS --enable-gtk-doc"
CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CFLAGS=-DARROW_NO_DEPRECATED_API"
CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CXXFLAGS=-DARROW_NO_DEPRECATED_API"
mkdir -p build
pushd build
../configure $CONFIGURE_OPTIONS
make -j4
make install
popd
rm -rf build
# Build with Meson
MESON_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL_MESON"
MESON_OPTIONS="$MESON_OPTIONS -Dgtk_doc=true"
mkdir -p build
env \
CFLAGS="-DARROW_NO_DEPRECATED_API" \
CXXFLAGS="-DARROW_NO_DEPRECATED_API" \
meson build $MESON_OPTIONS
pushd build
ninja
ninja install
popd
rm -rf build
popd
ci/travis_before_script_cpp.sh
-217
View File
@@ -1,217 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -ex
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
only_library_mode=no
using_homebrew=no
while true; do
case "$1" in
--only-library)
only_library_mode=yes
shift ;;
--homebrew)
using_homebrew=yes
shift ;;
*) break ;;
esac
done
if [ "$only_library_mode" == "no" ]; then
source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
fi
if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
# Set up C++ toolchain from conda-forge packages for faster builds
source $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
fi
mkdir -p $ARROW_CPP_BUILD_DIR
pushd $ARROW_CPP_BUILD_DIR
CMAKE_COMMON_FLAGS="\
-DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL \
-DARROW_WITH_BZ2=ON \
-DARROW_WITH_ZLIB=ON \
-DARROW_WITH_ZSTD=ON \
-DARROW_WITH_LZ4=ON \
-DARROW_WITH_SNAPPY=ON \
-DARROW_WITH_BROTLI=ON \
-DARROW_NO_DEPRECATED_API=ON \
-DARROW_INSTALL_NAME_RPATH=OFF \
-DARROW_EXTRA_ERROR_CONTEXT=ON"
CMAKE_LINUX_FLAGS=""
CMAKE_OSX_FLAGS="\
-DARROW_GFLAGS_USE_SHARED=OFF"
if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_JEMALLOC=ON"
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_WITH_BZ2=ON"
fi
if [ "$ARROW_TRAVIS_MIMALLOC" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_MIMALLOC=ON"
fi
if [ $only_library_mode == "yes" ]; then
CMAKE_COMMON_FLAGS="\
$CMAKE_COMMON_FLAGS \
-DARROW_BUILD_UTILITIES=OFF"
else
CMAKE_COMMON_FLAGS="\
$CMAKE_COMMON_FLAGS \
-DARROW_BUILD_BENCHMARKS=ON \
-DARROW_BUILD_TESTS=ON \
-DARROW_BUILD_EXAMPLES=ON \
-DARROW_BUILD_UTILITIES=ON"
fi
ARROW_CXXFLAGS=""
# Use Ninja for faster builds when using toolchain
if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -GNinja"
if [ "$DISTRO_CODENAME" != "trusty" ]; then
# Make sure the toolchain linker (from binutils package) is picked up by clang
ARROW_CXXFLAGS="$ARROW_CXXFLAGS -B$CPP_TOOLCHAIN/bin"
fi
export TRAVIS_MAKE=ninja
elif [ "$using_homebrew" = "yes" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -GNinja"
export TRAVIS_MAKE=ninja
else
export TRAVIS_MAKE=make
fi
if [ "$ARROW_TRAVIS_FLIGHT" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_FLIGHT=ON"
fi
if [ "$ARROW_TRAVIS_INTEGRATION" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_BUILD_INTEGRATION=ON"
fi
if [ "$ARROW_TRAVIS_PLASMA" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_PLASMA=ON"
fi
if [ "$ARROW_TRAVIS_PLASMA_JAVA_CLIENT" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_PLASMA_JAVA_CLIENT=ON"
fi
if [ "$ARROW_TRAVIS_ORC" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_ORC=ON"
fi
if [ "$ARROW_TRAVIS_PYTHON" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_PYTHON=ON"
if [ "$using_homebrew" == "yes" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS \
-DPYTHON_EXECUTABLE=$(brew --prefix python)/bin/python3"
fi
fi
if [ "$ARROW_TRAVIS_S3" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_S3=ON"
fi
if [ "$ARROW_TRAVIS_PARQUET" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS \
-DARROW_PARQUET=ON \
-DPARQUET_BUILD_EXAMPLES=ON \
-DPARQUET_BUILD_EXECUTABLES=ON"
fi
if [ "$ARROW_TRAVIS_PARQUET_ENCRYPTION" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DPARQUET_REQUIRE_ENCRYPTION=ON"
fi
if [ "$ARROW_TRAVIS_GANDIVA" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GANDIVA=ON"
if [ "$ARROW_TRAVIS_GANDIVA_JAVA" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GANDIVA_JAVA=ON"
fi
fi
if [ "$ARROW_TRAVIS_VALGRIND" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_TEST_MEMCHECK=ON"
fi
if [ "$ARROW_USE_ASAN" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_USE_ASAN=ON"
fi
if [ "$ARROW_USE_UBSAN" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_USE_UBSAN=ON"
fi
if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GENERATE_COVERAGE=ON"
fi
if [ "$ARROW_TRAVIS_VERBOSE" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_VERBOSE_THIRDPARTY_BUILD=ON"
fi
if [ "$ARROW_TRAVIS_STATIC_BOOST" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_BOOST_USE_SHARED=OFF"
fi
if [ "$ARROW_TRAVIS_OPTIONAL_INSTALL" == "1" ]; then
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_OPTIONAL_INSTALL=ON"
fi
if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
conda activate $CPP_TOOLCHAIN
fi
# conda-forge sets the build flags by default to -02, skip this to speed up the build
export CFLAGS=${CFLAGS//-O2}
export CXXFLAGS=${CXXFLAGS//-O2}
if [ $TRAVIS_OS_NAME == "osx" ]; then
source $TRAVIS_BUILD_DIR/ci/travis_install_osx_sdk.sh
fi
if [ $TRAVIS_OS_NAME == "linux" ]; then
cmake $CMAKE_COMMON_FLAGS \
$CMAKE_LINUX_FLAGS \
-DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
-DBUILD_WARNING_LEVEL=$ARROW_BUILD_WARNING_LEVEL \
-DARROW_CXXFLAGS="$ARROW_CXXFLAGS" \
$ARROW_CPP_DIR
else
cmake $CMAKE_COMMON_FLAGS \
$CMAKE_OSX_FLAGS \
-DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
-DBUILD_WARNING_LEVEL=$ARROW_BUILD_WARNING_LEVEL \
$ARROW_CPP_DIR
fi
# Build and install libraries. Configure ARROW_CPP_BUILD_TARGETS environment
# variable to only build certain targets. If you use this, you must also set
# the environment variable ARROW_TRAVIS_OPTIONAL_INSTALL=1
time $TRAVIS_MAKE -j4 $ARROW_CPP_BUILD_TARGETS
time $TRAVIS_MAKE install
popd
ci/travis_env_common.sh
-89
View File
@@ -1,89 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# hide nodejs experimental-feature warnings
export NODE_NO_WARNINGS=1
export MINICONDA=$HOME/miniconda
export CONDA_PKGS_DIRS=$HOME/.conda_packages
export CONDA_BINUTILS_VERSION=2.31
export ARROW_LLVM_VERSION=7.0
export CONDA_LLVM_VERSION="7.0.*"
# extract the major version
export ARROW_LLVM_MAJOR_VERSION=$(echo $ARROW_LLVM_VERSION | cut -d. -f1)
export ARROW_CPP_DIR=$TRAVIS_BUILD_DIR/cpp
export ARROW_PYTHON_DIR=$TRAVIS_BUILD_DIR/python
export ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib
export ARROW_JAVA_DIR=${TRAVIS_BUILD_DIR}/java
export ARROW_JS_DIR=${TRAVIS_BUILD_DIR}/js
export ARROW_INTEGRATION_DIR=$TRAVIS_BUILD_DIR/integration
export ARROW_DEV_DIR=$TRAVIS_BUILD_DIR/dev
export ARROW_CROSSBOW_DIR=$TRAVIS_BUILD_DIR/dev/tasks
export ARROW_RUBY_DIR=$TRAVIS_BUILD_DIR/ruby
export ARROW_GO_DIR=${TRAVIS_BUILD_DIR}/go
export ARROW_RUST_DIR=${TRAVIS_BUILD_DIR}/rust
export ARROW_R_DIR=${TRAVIS_BUILD_DIR}/r
export ARROW_TRAVIS_COVERAGE=${ARROW_TRAVIS_COVERAGE:=0}
if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then
export ARROW_CPP_COVERAGE_FILE=${TRAVIS_BUILD_DIR}/coverage.info
export ARROW_PYTHON_COVERAGE_FILE=${TRAVIS_BUILD_DIR}/.coverage
fi
export CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
export ARROW_C_GLIB_INSTALL_AUTOTOOLS=$TRAVIS_BUILD_DIR/c-glib-install-autotools
export ARROW_C_GLIB_INSTALL_MESON=$TRAVIS_BUILD_DIR/c-glib-install-meson
export CMAKE_EXPORT_COMPILE_COMMANDS=1
export ARROW_BUILD_TYPE=${ARROW_BUILD_TYPE:=debug}
export ARROW_BUILD_WARNING_LEVEL=${ARROW_BUILD_WARNING_LEVEL:=Production}
if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
# C++ toolchain
export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
fi
if [ $TRAVIS_OS_NAME == "osx" ]; then
export GOPATH=$TRAVIS_BUILD_DIR/gopath
fi
export ARROW_TEST_DATA=$TRAVIS_BUILD_DIR/testing/data
export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/cpp/submodules/parquet-testing/data
# e.g. "trusty" or "xenial"
if [ $TRAVIS_OS_NAME == "linux" ]; then
export DISTRO_CODENAME=`lsb_release -s -c`
fi
if [ "$ARROW_TRAVIS_USE_SYSTEM_JAVA" == "1" ]; then
# Use the Ubuntu-provided OpenJDK
unset JAVA_HOME
export TRAVIS_MVN=/usr/bin/mvn
export TRAVIS_JAVA=/usr/bin/java
else
export TRAVIS_MVN=mvn
export TRAVIS_JAVA=java
fi
ci/travis_install_conda.sh
-72
View File
@@ -1,72 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -e
if [ $TRAVIS_OS_NAME == "linux" ]; then
MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
else
MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
fi
function download_miniconda() {
wget --no-verbose -O miniconda.sh $MINICONDA_URL
}
export MINICONDA=$HOME/miniconda
export CONDA_PKGS_DIRS=$HOME/.conda_packages
# If miniconda is installed already, run the activation script
if [ -d "$MINICONDA" ]; then
source $MINICONDA/etc/profile.d/conda.sh
else
mkdir -p $CONDA_PKGS_DIRS
CONDA_RETRIES=5
until [ $CONDA_RETRIES -eq 0 ]
do
# If wget fails to resolve the repo.continuum.io host in the DNS, it will
# exit rather than retrying with backoff like other kinds of failures. We
# sleep for a few seconds and retry a number of times to reduce flakiness
download_miniconda && break
echo "Unable to connect to repo.continuum.io, waiting and then retrying"
CONDA_RETRIES=$[$CONDA_RETRIES - 1]
sleep 5
done
if [ $CONDA_RETRIES -eq 0 ]; then
# If we failed to download, try again so the error message will be visible
# in Travis CI logs
download_miniconda
fi
bash miniconda.sh -b -p $MINICONDA
source $MINICONDA/etc/profile.d/conda.sh
conda config --set auto_update_conda false
conda info -a
conda config --set show_channel_urls True
# Help with SSL timeouts to S3
conda config --set remote_connect_timeout_secs 12
conda config --add channels conda-forge
fi
conda info -a
Loaded 100 of 165 files, more files were not shown because too many files have changed in this diff. Show more