Cleanup files; separate JNI builds [skip ci]

.github/workflows/csharp.yml

@@ -28,8 +28,6 @@ on:
       - 'csharp/**'
 
 jobs:
-  # the docker container fails to run because of the ubuntu host versions, see
-  # https://github.com/dotnet/core/issues/3509
 
   all-platforms:
     name: AMD64 ${{ matrix.name }} C# ${{ matrix.dotnet }}
@@ -52,7 +50,7 @@ jobs:
             name: Windows 2019
             dotnet: 2.2.103
     steps:
-      - name: Install csharp
+      - name: Install C#
         uses: actions/setup-dotnet@v1
         with:
           dotnet-version: ${{ matrix.dotnet }}

.github/workflows/java-jni.yml

@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Java
+
+on:
+  push:
+    paths:
+      - 'ci/**'
+      - 'cpp/**'
+      - 'java/**'
+  pull_request:
+    paths:
+      - 'ci/**'
+      - 'cpp/**'
+      - 'java/**'
+
+jobs:
+
+  docker-jni:
+    name: AMD64 Debian 9 Java JNI and Plasma
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@v1
+        with:
+          submodules: true
+      - name: Docker Pull
+        run: |
+          docker-compose pull --ignore-pull-failures debian-java
+          docker-compose pull --ignore-pull-failures debian-java-jni
+      - name: Docker Build
+        run: |
+          docker-compose build debian-java
+          docker-compose build debian-java-jni
+      - name: Docker Run
+        run: |
+          docker-compose run debian-java-jni
+      - name: Docker Push
+        if: github.event_name == 'push' && github.ref == 'master'
+        shell: bash
+        run: |
+          docker login \
+            -u ${{ secrets.DOCKERHUB_USER }} \
+            -p ${{ secrets.DOCKERHUB_TOKEN }}
+          docker-compose push debian-java-jni

.github/workflows/java.yml

@@ -22,11 +22,9 @@ on:
     paths:
       - 'ci/**'
       - 'cpp/**'
-      - 'java/**'
   pull_request:
     paths:
       - 'ci/**'
-      - 'cpp/**'
       - 'java/**'
 
 jobs:
@@ -70,34 +68,6 @@ jobs:
             -p ${{ secrets.DOCKERHUB_TOKEN }}
           docker-compose push debian-java
 
-  docker-jni:
-    name: AMD64 Debian 9 Java JNI and Plasma
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Arrow
-        uses: actions/checkout@v1
-        with:
-          submodules: true
-      - name: Docker Pull
-        run: |
-          docker-compose pull --ignore-pull-failures debian-java
-          docker-compose pull --ignore-pull-failures debian-java-jni
-      - name: Docker Build
-        run: |
-          docker-compose build debian-java
-          docker-compose build debian-java-jni
-      - name: Docker Run
-        run: |
-          docker-compose run debian-java-jni
-      - name: Docker Push
-        if: github.event_name == 'push' && github.ref == 'master'
-        shell: bash
-        run: |
-          docker login \
-            -u ${{ secrets.DOCKERHUB_USER }} \
-            -p ${{ secrets.DOCKERHUB_TOKEN }}
-          docker-compose push debian-java-jni
-
   macos:
     name: AMD64 MacOS 10.15 Java JDK ${{ matrix.jdk }}
     runs-on: macos-latest

.github/workflows/js.yml

@@ -80,19 +80,7 @@ jobs:
         shell: bash
         run: ci/scripts/js_test.sh $(pwd)
 
-  # TODO(kszucs): windows build fails with:
-  #
-  # > NODE_NO_WARNINGS=1 gulp build
-  # 'NODE_NO_WARNINGS' is not recognized as an internal or external command,
-  # operable program or batch file.
-  # npm ERR! code ELIFECYCLE
-  # npm ERR! errno 1
-  # npm ERR! apache-arrow@1.0.0-SNAPSHOT build: `NODE_NO_WARNINGS=1 gulp build`
-  # npm ERR! Exit status 1
-  # npm ERR!
-  # npm ERR! Failed at the apache-arrow@1.0.0-SNAPSHOT build script.
-  # npm ERR! This is probably not a problem with npm. There is likely additional logging output above.
-  #
+  # TODO(kszucs): the windows build fails with platform speficic npm error
   # windows:
   #   name: AMD64 Windows 2019 NodeJS ${{ matrix.node }}
   #   runs-on: windows-latest

ci/conda_env_python.yml

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# don't add pandas here, because it is not a mandatory test dependency
 cython=0.29.7
 cloudpickle
 hypothesis

ci/docker/conda-latest-python-spark.dockerfile

@@ -30,10 +30,6 @@ RUN conda install -q \
         maven=${maven} && \
     conda clean --all
 
-# ENV MAVEN_HOME=/usr/local/maven \
-#     M2_HOME=/root/.m2 \
-#     PATH=/root/.m2/bin:/usr/local/maven/bin:$PATH
-
 # installing specific version of spark
 ARG spark=master
 RUN mkdir /spark && wget -q -O - https://github.com/apache/spark/archive/${spark}.tar.gz | tar -xzf - --strip-components=1 -C /spark

ci/docker/fedora-29-cpp.dockerfile

@@ -79,12 +79,3 @@ ENV CC=gcc \
     ARROW_GANDIVA_JAVA=ON \
     ARROW_BUILD_TESTS=ON \
     ARROW_HOME=/usr/local
-
-# Gandiva test is failing with:
-# Running gandiva-internals-test, redirecting output into /build/cpp/build/test-logs/gandiva-internals-test.txt (attempt 1/1)
-# 1364
-# : CommandLine Error: Option 'x86-experimental-vector-widening-legalization' registered more than once!
-# 1365
-# LLVM ERROR: inconsistency in registered CommandLine options
-# 1366
-# /build/cpp/src/gandiva

ci/scripts/integration_hiveserver2.sh

@@ -30,5 +30,3 @@ pushd ${build_dir}
 debug/hiveserver2-test
 
 popd
-
-

ci/scripts/python_benchmark.sh

@@ -17,8 +17,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# TODO(kszucs): port it
-
 # Check the ASV benchmarking setup.
 # Unfortunately this won't ensure that all benchmarks succeed
 # (see https://github.com/airspeed-velocity/asv/issues/449)

docker-compose.yml

@@ -648,7 +648,9 @@ services:
         arch: ${ARCH}
         conda: latest
         jdk: ${JDK}
-        maven: ${MAVEN}
+        # conda-forge doesn't have 3.5.4 so pinning explicitly, but this should
+        # be set to ${MAVEN}
+        maven: 3.5
         node: ${NODE}
         go: ${GO}
     volumes: *conda-volumes
@@ -816,7 +818,10 @@ services:
       - ARROW_HDFS_TEST_PORT=8020
       - ARROW_HDFS_TEST_USER=hdfs
     shm_size: *shm-size
-    volumes: *conda-volumes
+    volumes: &conda-maven-volumes
+      - .:/arrow:delegated
+      - conda-latest-cache:/build:delegated
+      - maven-cache:/root/.m2:delegated
     entrypoint: *conda-entrypoint
     command:
       ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
@@ -846,9 +851,7 @@ services:
         maven: 3.5
         spark: ${SPARK}
     shm_size: *shm-size
-    volumes: *conda-volumes
-    # - maven-cache:/root/.m2:delegated
-    # - spark-cache:/spark:delegated
+    volumes: *conda-maven-volumes
     entrypoint: *conda-entrypoint
     command:
       ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&