ARROW-345: Proper locations of libhdfs and libjvm on Mac

cpp/doc/HDFS.md

@@ -43,7 +43,7 @@ LD_LIBRARY_PATH), and relies on some environment variables.
 export CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath --glob`
 ```
 
-#### Setting $JAVA_HOME  automatically on OS X
+### Mac Specifics
 
 The installed location of Java on OS X can vary, however the following snippet
 will set it automatically for you:
@@ -51,3 +51,23 @@ will set it automatically for you:
 ```shell
 export JAVA_HOME=$(/usr/libexec/java_home)
 ```
+
+Homebrew's Hadoop does not have native libs. Apache doesn't build these, so
+users must build Hadoop to get the native libs. See this Stack Overflow
+answer for details:
+
+http://stackoverflow.com/a/40051353/478288
+
+Be sure to include the path to the native libs in `JAVA_LIBRARY_PATH`:
+
+```shell
+export JAVA_LIBRARY_PATH=$HADOOP_HOME/lib/native:$JAVA_LIBRARY_PATH
+```
+
+If you get an error about needing to install Java 6, then add *BundledApp* and
+*JNI* to the `JVMCapabilities` in `$JAVA_HOME/../Info.plist`. See
+
+https://oliverdowling.com.au/2015/10/09/oracles-jre-8-on-mac-os-x-el-capitan/
+
+https://derflounder.wordpress.com/2015/08/08/modifying-oracles-java-sdk-to-run-java-applications-on-os-x/
+

cpp/src/arrow/io/libhdfs_shim.cc

@@ -342,18 +342,36 @@ int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime) {
 }
 
 static std::vector<fs::path> get_potential_libhdfs_paths() {
-  std::vector<fs::path> libhdfs_potential_paths = {
-      // find one in the local directory
-      fs::path("./libhdfs.so"), fs::path("./hdfs.dll"),
-      // find a global libhdfs.so
-      fs::path("libhdfs.so"), fs::path("hdfs.dll"),
+  std::vector<fs::path> libhdfs_potential_paths;
+  std::string file_name;
+
+  // OS-specific file name
+#ifdef __WIN32
+  file_name = "hdfs.dll";
+#elif __APPLE__
+  file_name = "libhdfs.dylib";
+#else
+  file_name = "libhdfs.so";
+#endif
+
+  // Common paths
+  std::vector<fs::path> search_paths = {
+      fs::path(""),
+      fs::path(".")
   };
 
+  // Path from environment variable
   const char* hadoop_home = std::getenv("HADOOP_HOME");
   if (hadoop_home != nullptr) {
-    auto path = fs::path(hadoop_home) / "lib/native/libhdfs.so";
-    libhdfs_potential_paths.push_back(path);
+    auto path = fs::path(hadoop_home) / "lib/native";
+    search_paths.push_back(path);
   }
+
+  // All paths with file name
+  for (auto& path : search_paths) {
+    libhdfs_potential_paths.push_back(path / file_name);
+  }
+
   return libhdfs_potential_paths;
 }
 
@@ -371,7 +389,7 @@ static std::vector<fs::path> get_potential_libjvm_paths() {
   file_name = "jvm.dll";
 #elif __APPLE__
   search_prefixes = {""};
-  search_suffixes = {""};
+  search_suffixes = {"", "/jre/lib/server"};
   file_name = "libjvm.dylib";
 
 // SFrame uses /usr/libexec/java_home to find JAVA_HOME; for now we are