Add arrow-orc setup

setup basic development for native orc reader

cpp/CMakeLists.txt

@@ -252,10 +252,16 @@ endif()
 if(MSVC)
   # ORC doesn't build on windows
   set(ARROW_ORC OFF)
+  set(ARROW_ORC_JNI OFF)
   # Plasma using glog is not fully tested on windows.
   set(ARROW_USE_GLOG OFF)
 endif()
 
+if(ARROW_ORC_JNI)
+  set(ARROW_ORC ON)
+  set(ARROW_BUILD_STATIC ON)
+endif()
+
 if(ARROW_ORC)
   set(ARROW_WITH_LZ4 ON)
   set(ARROW_WITH_SNAPPY ON)

cpp/cmake_modules/BuildUtils.cmake

@@ -139,7 +139,8 @@ function(ADD_ARROW_LIB LIB_NAME)
       PRIVATE_INCLUDES
       DEPENDENCIES
       SHARED_INSTALL_INTERFACE_LIBS
-      STATIC_INSTALL_INTERFACE_LIBS)
+      STATIC_INSTALL_INTERFACE_LIBS
+      OUTPUT_PATH)
   cmake_parse_arguments(ARG
                         "${options}"
                         "${one_value_args}"
@@ -164,6 +165,11 @@ function(ADD_ARROW_LIB LIB_NAME)
   else()
     set(BUILD_STATIC ${ARROW_BUILD_STATIC})
   endif()
+  if(ARG_OUTPUT_PATH)
+    set(OUTPUT_PATH ${ARG_OUTPUT_PATH})
+  else()
+    set(OUTPUT_PATH ${BUILD_OUTPUT_ROOT_DIRECTORY})
+  endif()
 
   if(WIN32 OR (CMAKE_GENERATOR STREQUAL Xcode))
     # We need to compile C++ separately for each library kind (shared and static)
@@ -234,11 +240,11 @@ function(ADD_ARROW_LIB LIB_NAME)
 
     set_target_properties(${LIB_NAME}_shared
                           PROPERTIES LIBRARY_OUTPUT_DIRECTORY
-                                     "${BUILD_OUTPUT_ROOT_DIRECTORY}"
+                                     "${OUTPUT_PATH}"
                                      RUNTIME_OUTPUT_DIRECTORY
-                                     "${BUILD_OUTPUT_ROOT_DIRECTORY}"
+                                     "${OUTPUT_PATH}"
                                      PDB_OUTPUT_DIRECTORY
-                                     "${BUILD_OUTPUT_ROOT_DIRECTORY}"
+                                     "${OUTPUT_PATH}"
                                      LINK_FLAGS
                                      "${ARG_SHARED_LINK_FLAGS}"
                                      OUTPUT_NAME
@@ -314,7 +320,7 @@ function(ADD_ARROW_LIB LIB_NAME)
 
     set_target_properties(${LIB_NAME}_static
                           PROPERTIES LIBRARY_OUTPUT_DIRECTORY
-                                     "${BUILD_OUTPUT_ROOT_DIRECTORY}" OUTPUT_NAME
+                                     "${OUTPUT_PATH}" OUTPUT_NAME
                                      ${LIB_NAME_STATIC})
 
     if(ARG_STATIC_INSTALL_INTERFACE_LIBS)

cpp/src/arrow/adapters/orc/CMakeLists.txt

@@ -49,3 +49,7 @@ add_arrow_test(adapter-test
                "orc"
                STATIC_LINK_LIBS
                ${ORC_STATIC_TEST_LINK_LIBS})
+
+if(ARROW_ORC_JNI)
+  add_subdirectory(jni)
+endif()

cpp/src/arrow/adapters/orc/jni/CMakeLists.txt

@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#
+# arrow_orc_jni
+#
+
+project(arrow_orc_jni)
+
+cmake_minimum_required(VERSION 3.11)
+
+find_package(JNI REQUIRED)
+
+add_custom_target(arrow_orc_jni)
+
+set(JNI_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated")
+
+add_subdirectory(../../../../../../java/adapter/orc ./java)
+
+set(ARROW_BUILD_STATIC OFF)
+
+ADD_ARROW_LIB(arrow_orc_jni
+        BUILD_SHARED
+        SOURCES
+        jni_wrapper.cpp
+        OUTPUTS
+        ARROW_ORC_JNI_LIBRARIES
+        SHARED_PRIVATE_LINK_LIBS
+        arrow_static
+        EXTRA_INCLUDES
+        ${JNI_HEADERS_DIR}
+        PRIVATE_INCLUDES
+        ${JNI_INCLUDE_DIRS}
+        ${CMAKE_CURRENT_BINARY_DIR}
+        DEPENDENCIES
+        arrow_static
+        arrow_orc_java
+        OUTPUT_PATH
+        ${CMAKE_CURRENT_BINARY_DIR})
+
+add_dependencies(arrow_orc_jni ${ARROW_ORC_JNI_LIBRARIES})

cpp/src/arrow/adapters/orc/jni/jni_wrapper.cpp

@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <string>
+#include <arrow/io/api.h>
+
+#include "org_apache_arrow_adapter_orc_OrcReaderJniWrapper.h"
+#include "org_apache_arrow_adapter_orc_OrcReaderJniWrapper_StripeReader.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+JNIEXPORT jboolean JNICALL Java_org_apache_arrow_adapter_orc_OrcReaderJniWrapper_open
+        (JNIEnv *env, jobject this_obj, jstring file_path) {
+    auto in_file = new std::shared_ptr<arrow::io::ReadableFile>();
+    const char *str= env->GetStringUTFChars(file_path, nullptr);
+    std::string path(str);
+    env->ReleaseStringUTFChars(file_path, str);
+
+    arrow::Status ret;
+    if (path.find("hdfs://") == 0) {
+        return false;
+    } else {
+        ret = arrow::io::ReadableFile::Open(path, in_file);
+    }
+
+    if(ret.ok()) {
+
+        jclass this_class = env->GetObjectClass(this_obj);
+        jfieldID fidNativeManager = env->GetFieldID(this_class, "nativeReaderAddress", "J");
+        if (fidNativeManager == nullptr)
+        {
+            return false;
+        }
+
+        env->SetLongField(this_obj, fidNativeManager, (long)in_file);
+    }
+
+    return ret.ok();
+}
+
+#ifdef __cplusplus
+}
+#endif

java/README.md

@@ -45,6 +45,15 @@ mvn install -P gandiva -pl gandiva -am -Dgandiva.cpp.build.dir=../../debug
 This library is still in Alpha stages, and subject to API changes without
 deprecation warnings.
 
+## Building and running tests for native orc (optional)
+Arrow Cpp must be built before this step. The cpp build directory must
+be provided as the value for argument arrow.cpp.build.dir. eg.
+
+```
+cd java
+mvn install -P native-orc -pl adapter/orc -am -Dgandiva.cpp.build.dir=../../debug
+```
+
 ## Java Code Style Guide
 
 Arrow Java follows the Google style guide [here][3] with the following

java/adapter/orc/CMakeLists.txt

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#
+# arrow_orc_java
+#
+
+# Headers: top level
+
+project(arrow_orc_java)
+
+# Find java/jni
+include(FindJava)
+include(UseJava)
+include(FindJNI)
+
+message("generating headers to ${JNI_HEADERS_DIR}")
+
+add_jar(
+        arrow_orc_java
+        src/main/java/org/apache/arrow/adapter/orc/OrcReaderJniWrapper.java
+        GENERATE_NATIVE_HEADERS arrow_orc_java-native
+        DESTINATION ${JNI_HEADERS_DIR}
+)

java/adapter/orc/pom.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+    license agreements. See the NOTICE file distributed with this work for additional
+    information regarding copyright ownership. The ASF licenses this file to
+    You under the Apache License, Version 2.0 (the "License"); you may not use
+    this file except in compliance with the License. You may obtain a copy of
+    the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+    by applicable law or agreed to in writing, software distributed under the
+    License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+    OF ANY KIND, either express or implied. See the License for the specific
+    language governing permissions and limitations under the License. -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.apache.arrow</groupId>
+        <artifactId>arrow-java-root</artifactId>
+        <version>0.14.0-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+
+    <groupId>org.apache.arrow.orc</groupId>
+    <artifactId>arrow-orc</artifactId>
+    <name>Arrow Orc Adapter</name>
+    <packaging>jar</packaging>
+    <properties>
+        <arrow.cpp.build.dir>../../../cpp/debug-build/</arrow.cpp.build.dir>
+    </properties>
+
+    <build>
+        <resources>
+            <resource>
+                <directory>${arrow.cpp.build.dir}/src/arrow/adapters/orc/jni/</directory>
+                <includes>
+                    <include>**/libarrow_orc_jni.*</include>
+                </includes>
+            </resource>
+        </resources>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <systemPropertyVariables>
+                        <user.timezone>UTC</user.timezone>
+                    </systemPropertyVariables>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReaderJniWrapper.java

@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.adapter.orc;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
+import java.nio.file.StandardCopyOption;
+import java.util.UUID;
+
+public class OrcReaderJniWrapper {
+  private static final String LIBRARY_NAME = "arrow_orc_jni";
+
+  private long nativeReaderAddress;
+
+  static {
+    try {
+      loadOrcAdapterLibraryFromJar();
+    } catch (IOException e) {
+      throw new ExceptionInInitializerError(e);
+    }
+  }
+
+  private static void loadOrcAdapterLibraryFromJar()
+          throws IOException {
+    final String libraryToLoad = System.mapLibraryName(LIBRARY_NAME);
+    final File libraryFile = moveFileFromJarToTemp(
+            System.getProperty("java.io.tmpdir"), libraryToLoad);
+    System.load(libraryFile.getAbsolutePath());
+  }
+
+
+  private static File moveFileFromJarToTemp(final String tmpDir, String libraryToLoad)
+          throws IOException {
+    final File temp = setupFile(tmpDir, libraryToLoad);
+    try (final InputStream is = OrcReaderJniWrapper.class.getClassLoader()
+            .getResourceAsStream(libraryToLoad)) {
+      if (is == null) {
+        throw new InvalidPathException(libraryToLoad,  "file was not found inside JAR.");
+      } else {
+        Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING);
+      }
+    }
+    return temp;
+  }
+
+  private static File setupFile(String tmpDir, String libraryToLoad)
+          throws IOException {
+    final String randomizeFileName = libraryToLoad + UUID.randomUUID();
+    final File temp = new File(tmpDir, randomizeFileName);
+    if (temp.exists() && !temp.delete()) {
+      throw new InvalidPathException(
+              temp.getAbsolutePath(), "File already exists and cannot be removed.");
+    }
+    if (!temp.createNewFile()) {
+      throw new InvalidPathException(temp.getAbsolutePath(), "File could not be created.");
+    }
+    temp.deleteOnExit();
+    return temp;
+  }
+
+  private native boolean open(String fileName);
+
+  public native void close();
+
+  public native boolean seek(int rowNumber);
+
+  public native int getNumberOfStripes();
+
+  public native StripeReader nextStripeReader(long batchSize);
+
+  public class StripeReader {
+
+    private long nativeStripeReaderAddress;
+
+    public native byte[] getSchema();
+
+    public native boolean next(int numRows, long[] outAddrs, long[] outSizes);
+  }
+}

java/pom.xml

@@ -682,6 +682,21 @@
         <module>gandiva</module>
       </modules>
     </profile>
+
+    <profile>
+      <!-- orc java depends on arrow cpp, and arrow cpp isn't enabled by default yet -->
+      <id>native-orc</id>
+      <modules>
+        <module>format</module>
+        <module>memory</module>
+        <module>vector</module>
+        <module>tools</module>
+        <module>adapter/jdbc</module>
+        <module>adapter/orc</module>
+        <module>plasma</module>
+        <module>flight</module>
+      </modules>
+    </profile>
   </profiles>
 
 </project>