ARROW-4909: [CI] Use hadolint to lint Dockerfiles

.hadolint.yaml

@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ignored:
+  - DL3008
+  - DL3013
+  - DL3018

.pre-commit-config.yaml

@@ -43,15 +43,24 @@ repos:
         entry: bash -c "pip install cmake-format && python run-cmake-format.py --check"
         entry: echo
         files: ^(.*/CMakeLists.txt|.*.cmake)$
+      - id: hadolint
+        name: hadolint
+        language: docker_image
+        types:
+         - dockerfile
+        entry: --entrypoint /bin/hadolint hadolint/hadolint:latest -
+        exclude: ^dev/.*$
   - repo: git://github.com/pre-commit/pre-commit-hooks
     sha: v1.2.3
     hooks:
      - id: flake8
+       name: Flake8 Python
        files: ^(python|crossbow|integration)/
        types:
          - file
          - python
      - id: flake8
+       name: Flake8 Cython
        files: ^python/
        types:
          - file

.travis.yml

@@ -46,7 +46,7 @@ before_install:
 matrix:
   fast_finish: true
   include:
-  - name: "Lint C++, Python, R"
+  - name: "Lint C++, Python, R, Docker"
     os: linux
     language: python
     python: "3.6"

c_glib/Dockerfile

@@ -17,14 +17,19 @@
 
 FROM arrow:cpp
 
-RUN apt-get -q install --no-install-recommends -y \
-        ruby-dev \
-        pkg-config \
+RUN apt-get update -y -q && \
+    apt-get -q install --no-install-recommends -y \
         autoconf-archive \
+        gobject-introspection \
         gtk-doc-tools \
-        libgirepository1.0-dev
+        libgirepository1.0-dev \
+        libglib2.0-dev \
+        pkg-config \
+        ruby-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
-ADD c_glib/Gemfile /arrow/c_glib/
+COPY c_glib/Gemfile /arrow/c_glib/
 RUN conda install meson=0.47.1 && \
     conda clean --all && \
     gem install bundler && \
@@ -39,6 +44,6 @@ ENV ARROW_BUILD_TESTS=OFF \
     GI_TYPELIB_PATH="${CONDA_PREFIX}/lib/girepository-1.0"
 
 # build, install and test
-CMD arrow/ci/docker_build_cpp.sh && \
+CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && \
     arrow/ci/docker_build_c_glib.sh && \
-    arrow/c_glib/test/run-test.rb
+    arrow/c_glib/test/run-test.rb"]

ci/docker_build_and_test_cpp.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+/arrow/ci/docker_build_cpp.sh
+pushd /build/cpp
+ninja test
+popd

ci/docker_build_and_test_python.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+/arrow/ci/docker_build_cpp.sh
+/arrow/ci/docker_build_python.sh
+pytest -v --pyargs pyarrow

ci/travis_lint.sh

@@ -23,6 +23,13 @@ set -ex
 export ARROW_TRAVIS_USE_TOOLCHAIN=0
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
+pip install pre_commit
+pre-commit install
+
+# TODO: Move more checks into pre-commit as this gives a nice summary
+# and doesn't abort on the first failed check.
+pre-commit run hadolint -a
+
 # CMake formatting check
 pip install cmake_format
 $TRAVIS_BUILD_DIR/run-cmake-format.py --check

cpp/Dockerfile

@@ -29,16 +29,18 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         ninja-build \
         pkg-config \
         tzdata \
-        wget
+        wget \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # install conda and required packages
 ARG EXTRA_CONDA_PKGS
 ENV PATH=/opt/conda/bin:$PATH \
     CONDA_PREFIX=/opt/conda
-ADD ci/docker_install_conda.sh \
-    ci/conda_env_cpp.yml \
-    ci/conda_env_unix.yml \
-    /arrow/ci/
+COPY ci/docker_install_conda.sh \
+     ci/conda_env_cpp.yml \
+     ci/conda_env_unix.yml \
+     /arrow/ci/
 RUN arrow/ci/docker_install_conda.sh && \
     conda install -q -c conda-forge \
         --file arrow/ci/conda_env_cpp.yml \
@@ -55,5 +57,4 @@ ENV CC=gcc \
     PARQUET_HOME=$CONDA_PREFIX
 
 # build and test
-CMD arrow/ci/docker_build_cpp.sh && \
-    cd /build/cpp && ninja test
+CMD ["arrow/ci/docker_build_and_test_cpp.sh"]

cpp/Dockerfile.alpine

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM alpine
+FROM alpine:3.9
 
 # Install basic build dependencies
 # grpc requires libnsl-dev to be present, this cannot be installed via a bundle.
@@ -41,14 +41,14 @@ RUN apk add --no-cache -q \
 
 # Ganidva is deactivated as we don't support building LLVM via ExternalProject
 # and Alpine only has LLVM 6 in its repositories yet.
+# ARROW-4917: ORC fails with compiler problems
 ENV CC=gcc \
     CXX=g++ \
     ARROW_GANDIVA=OFF \
-    ARROW_ORC=ON \
+    ARROW_ORC=OFF \
     ARROW_PARQUET=ON \
     ARROW_BUILD_TESTS=ON \
     ARROW_HOME=/usr/local
 
 # build and test
-CMD arrow/ci/docker_build_cpp.sh && \
-    cd /build/cpp && ninja test
+CMD ["arrow/ci/docker_build_and_test_cpp.sh"]

cpp/Dockerfile.debian-testing

@@ -57,7 +57,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         rapidjson-dev \
         thrift-compiler \
         tzdata \
-        wget
+        wget \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 ENV CC=gcc \
      CXX=g++ \
@@ -71,6 +73,4 @@ ENV CC=gcc \
      CMAKE_ARGS="-DFlatbuffers_SOURCE=BUNDLED -Dc-ares_SOURCE=BUNDLED -DgRPC_SOURCE=BUNDLED"
 
 # build and test
-CMD arrow/ci/docker_build_cpp.sh && \
-    cd /build/cpp && ctest -j2 --output-on-failure -L unittest
-
+CMD ["arrow/ci/docker_build_and_test_cpp.sh"]

cpp/Dockerfile.fedora

@@ -73,4 +73,4 @@ ENV CC=gcc \
      CMAKE_ARGS="-DgRPC_SOURCE=BUNDLED"
 
 # build; tests don't work on Fedora at the moment due to compiler problems in gcc 8.2 (fixed in 8.3)
-CMD arrow/ci/docker_build_cpp.sh
+CMD ["arrow/ci/docker_build_cpp.sh"]

cpp/Dockerfile.ubuntu-bionic

@@ -17,13 +17,13 @@
 
 FROM ubuntu:bionic
 
-# install build essentials
-RUN export DEBIAN_FRONTEND=noninteractive && \
-    apt-get update -y -q && \
-    apt-get install -y wget software-properties-common
-RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
-RUN apt-add-repository -y "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main"
-RUN export DEBIAN_FRONTEND=noninteractive && \
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends wget software-properties-common gpg-agent && \
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    apt-add-repository -y "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main" && \
     apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
         autoconf \
@@ -60,7 +60,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         protobuf-compiler \
         protobuf-compiler-grpc \
         rapidjson-dev \
-        tzdata
+        tzdata \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # Ubuntu's gtest just provides sources, the compiled version is only available
 # from Ubuntu Cosmic on.
@@ -83,6 +85,4 @@ ENV CC=gcc \
      CMAKE_ARGS="-DThrift_SOURCE=BUNDLED -DFlatbuffers_SOURCE=BUNDLED -DGTest_SOURCE=BUNDLED -DgRPC_SOURCE=BUNDLED -Dc-ares_SOURCE=BUNDLED -DProtobuf_SOURCE=BUNDLED"
 
 # build and test
-CMD arrow/ci/docker_build_cpp.sh && \
-    cd /build/cpp && ctest -j2 --output-on-failure -L unittest
-
+CMD ["arrow/ci/docker_build_and_test_cpp.sh"]

cpp/Dockerfile.ubuntu-xenial

@@ -17,13 +17,13 @@
 
 FROM ubuntu:xenial
 
-# install build essentials
-RUN export DEBIAN_FRONTEND=noninteractive && \
-    apt-get update -y -q && \
-    apt-get install -y wget software-properties-common
-RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
-RUN apt-add-repository -y "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main"
-RUN export DEBIAN_FRONTEND=noninteractive && \
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends wget software-properties-common && \
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    apt-add-repository -y "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main" && \
     apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
         autoconf \
@@ -44,6 +44,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         libgflags-dev \
         libgoogle-glog-dev \
         liblz4-dev \
+        libprotoc-dev \
         libprotobuf-dev \
         libre2-dev \
         libsnappy-dev \
@@ -54,7 +55,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         ninja-build \
         pkg-config \
         protobuf-compiler \
-        tzdata
+        tzdata \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # Benchmark is deactivated as the external project requires CMake 3.6+
 # Flight is deactivated as Ubuntu provides gflags but not the CMake config for it.
@@ -74,5 +77,4 @@ ENV CC=gcc \
      CMAKE_ARGS="-DThrift_SOURCE=BUNDLED -DGTest_SOURCE=BUNDLED -DFlatbuffers_SOURCE=BUNDLED -DRapidJSON_SOURCE=BUNDLED"
 
 # build and test
-CMD arrow/ci/docker_build_cpp.sh && \
-    cd /build/cpp && ctest -j2 --output-on-failure -L unittest
+CMD ["arrow/ci/docker_build_and_test_cpp.sh"]

csharp/build/docker/Dockerfile

@@ -15,9 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM microsoft/dotnet/2.1-sdk
+FROM mcr.microsoft.com/dotnet/core/sdk:2.1
 
-ADD csharp /arrow/csharp
+COPY csharp /arrow/csharp
 WORKDIR /arrow/csharp
 
-CMD dotnet build
+CMD ["dotnet", "build"]

docs/Dockerfile

@@ -17,10 +17,8 @@
 
 FROM arrow:python-3.6
 
-ADD ci/conda_env_sphinx.yml /arrow/ci/
+COPY ci/conda_env_sphinx.yml /arrow/ci/
 RUN conda install --file arrow/ci/conda_env_sphinx.yml && \
     conda clean --all
 
-CMD arrow/ci/docker_build_cpp.sh && \
-    arrow/ci/docker_build_python.sh && \
-    arrow/ci/docker_build_sphinx.sh
+CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && arrow/ci/docker_build_python.sh && arrow/ci/docker_build_sphinx.sh"]

go/Dockerfile

@@ -15,14 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM golang
+FROM golang:1.12
 
-ADD go/arrow/Gopkg.lock \
-    go/arrow/Gopkg.toml \
-    /arrow/go/arrow/
+COPY go/arrow/Gopkg.lock \
+     go/arrow/Gopkg.toml \
+     /arrow/go/arrow/
 WORKDIR /arrow/go/arrow
 
 RUN go get -d -t -v ./...
 
-CMD go install -v ./... && \
-    for d in $(go list ./... | grep -v vendor); do go test $d; done
+CMD ["/bin/bash", "-c", "go install -v ./... && for d in $(go list ./... | grep -v vendor); do go test $d; done"]

integration/dask/Dockerfile

@@ -17,15 +17,11 @@
 
 FROM arrow:python-3.6
 
-# setup /etc/localtime
-RUN DEBIAN_FRONTEND=noninteractive \
-    apt-get install -y -q tzdata
-
 # install dask release from conda
 RUN conda install -c conda-forge dask pytest=3 && \
     conda clean --all
 
 # build and test
-CMD arrow/ci/docker_build_cpp.sh && \
+CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && \
     arrow/ci/docker_build_python.sh && \
-    arrow/integration/dask/runtest.sh
+    arrow/integration/dask/runtest.sh"]

integration/hdfs/Dockerfile

@@ -24,12 +24,14 @@ ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \
     HADOOP_OPTS=-Djava.library.path=/usr/local/hadoop/lib/native \
     PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
 RUN apt-get update -y && \
-    apt-get install -y openjdk-8-jdk && \
+    apt-get install -y --no-install-recommends openjdk-8-jdk && \
     wget -q -O hadoop-$HADOOP_VERSION.tar.gz "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz" && \
     tar -zxf /hadoop-$HADOOP_VERSION.tar.gz && \
     rm /hadoop-$HADOOP_VERSION.tar.gz && \
-    mv /hadoop-$HADOOP_VERSION /usr/local/hadoop
-ADD integration/hdfs/hdfs-site.xml $HADOOP_HOME/etc/hadoop/
+    mv /hadoop-$HADOOP_VERSION /usr/local/hadoop \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+COPY integration/hdfs/hdfs-site.xml $HADOOP_HOME/etc/hadoop/
 
 # installing libhdfs3, it needs to be pinned, see ARROW-1465 and ARROW-1445
 # after the conda-forge migration it's failing with abi incompatibilities, so
@@ -46,6 +48,6 @@ ENV CC=gcc \
     ARROW_BUILD_TESTS=ON
 
 # build and test
-CMD arrow/ci/docker_build_cpp.sh && \
+CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && \
     arrow/ci/docker_build_python.sh && \
-    arrow/integration/hdfs/runtest.sh
+    arrow/integration/hdfs/runtest.sh"]

integration/spark/Dockerfile

@@ -23,11 +23,13 @@ ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \
     M2_HOME=/root/.m2 \
     PATH=/root/.m2/bin:/usr/local/maven/bin:$PATH
 RUN apt-get update -q -y && \
-    apt-get install -q -y openjdk-8-jdk && \
+    apt-get install -q -y --no-install-recommends openjdk-8-jdk && \
     wget -q -O maven-$MAVEN_VERSION.tar.gz "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz" && \
     tar -zxf /maven-$MAVEN_VERSION.tar.gz && \
     rm /maven-$MAVEN_VERSION.tar.gz && \
-    mv /apache-maven-$MAVEN_VERSION /usr/local/maven
+    mv /apache-maven-$MAVEN_VERSION /usr/local/maven \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # installing specific version of spark
 ARG SPARK_VERSION=master
@@ -44,7 +46,7 @@ ENV CC=gcc \
     ARROW_BUILD_TESTS=OFF
 
 # build and test
-CMD arrow/ci/docker_build_cpp.sh && \
+CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && \
     arrow/ci/docker_build_python.sh && \
     arrow/ci/docker_build_java.sh && \
-    arrow/integration/spark/runtest.sh
+    arrow/integration/spark/runtest.sh"]

java/Dockerfile

@@ -19,8 +19,11 @@ FROM maven:3.5.2-jdk-8-slim
 
 # rsync is required to prevent the contamination of arrow directory
 # (mounted from the host)
-RUN apt-get update -y && apt-get install -y rsync
+RUN apt-get update -y \
+    && apt-get install -y --no-install-recommends rsync \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
-CMD arrow/ci/docker_build_java.sh && \
+CMD ["/bin/bash", "-c", "arrow/ci/docker_build_java.sh && \
     cd /build/java/arrow/java && \
-    mvn test
+    mvn test"]

js/Dockerfile

@@ -15,18 +15,18 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM node
+FROM node:11
 
 # install dependencies
-ADD js/.npmrc js/package.json /arrow/js/
+COPY js/.npmrc js/package.json /arrow/js/
 WORKDIR /arrow/js
 RUN npm install -g npm@latest && \
     npm install
 
 # build
-ADD LICENSE.txt /arrow/
-ADD NOTICE.txt /arrow/
-ADD js /arrow/js
+COPY LICENSE.txt /arrow/
+COPY NOTICE.txt /arrow/
+COPY js /arrow/js
 RUN npm run lint && npm run build
 
-CMD npm run test
+CMD ["npm", "run", "test"]

python/Dockerfile

@@ -19,7 +19,7 @@ FROM arrow:cpp
 
 # install python specific packages
 ARG PYTHON_VERSION=3.6
-ADD ci/conda_env_python.yml /arrow/ci/
+COPY ci/conda_env_python.yml /arrow/ci/
 RUN conda install -q \
         --file arrow/ci/conda_env_python.yml \
         python=$PYTHON_VERSION \
@@ -30,6 +30,4 @@ ENV ARROW_PYTHON=ON \
     ARROW_BUILD_TESTS=OFF
 
 # build and test
-CMD arrow/ci/docker_build_cpp.sh && \
-    arrow/ci/docker_build_python.sh && \
-    pytest -v --pyargs pyarrow
+CMD ["arrow/ci/docker_build_and_test_python.sh"]

python/Dockerfile.alpine

@@ -17,6 +17,8 @@
 
 FROM arrow:cpp-alpine
 
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
 # better compatibility for the scripts
 RUN apk add --no-cache coreutils
 
@@ -30,9 +32,9 @@ RUN export PYTHON_MAJOR=${PYTHON_VERSION:0:1} && \
     pip install --upgrade pip setuptools
 
 # install python requirements
-ADD python/requirements.txt \
-    python/requirements-test.txt \
-    /arrow/python/
+COPY python/requirements.txt \
+     python/requirements-test.txt \
+     /arrow/python/
 # pandas requires numpy at build time, so install the requirements separately
 RUN pip install -r /arrow/python/requirements.txt cython && \
     pip install -r /arrow/python/requirements-test.txt
@@ -42,6 +44,4 @@ ENV ARROW_PYTHON=ON \
     PYARROW_WITH_PARQUET=0
 
 # build and test
-CMD arrow/ci/docker_build_cpp.sh && \
-    arrow/ci/docker_build_python.sh && \
-    pytest -v --pyargs pyarrow
+CMD ["arrow/ci/docker_build_and_test_python.sh"]

r/Dockerfile

@@ -72,5 +72,5 @@ ENV PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/build/cpp/src/arrow:/opt/conda/lib/pkgco
 ENV LD_LIBRARY_PATH=/opt/conda/lib/:/build/cpp/src/arrow:/arrow/r/src
 
 # build, install, test R package
-CMD /arrow/ci/docker_build_cpp.sh && \
-    /arrow/ci/docker_build_r.sh
+CMD ["/bin/bash", "-c", "/arrow/ci/docker_build_cpp.sh && \
+    /arrow/ci/docker_build_r.sh"]

rust/Dockerfile

@@ -20,7 +20,7 @@ FROM rustlang/rust:nightly
 # sadly cargo doesn't have a command to fetch and build the
 # dependencies without building the library itself
 ENV CARGO_TARGET_DIR=/build/rust
-ADD rust/Cargo.toml /arrow/rust/Cargo.toml
+COPY rust/Cargo.toml /arrow/rust/Cargo.toml
 WORKDIR /arrow/rust
 
-CMD cargo test
+CMD ["cargo", "test"]