ARROW-7282: [Python] IO functions should raise the right exceptions

When an Arrow IO function fails, it should be turned into the appropriate
Python exception subclass, e.g. FileNotFoundError in case the requested
file or directory is missing.

cpp/src/arrow/filesystem/localfs.cc

@@ -17,6 +17,7 @@
 
 #include <chrono>
 #include <cstring>
+#include <sstream>
 #include <utility>
 
 #ifdef _WIN32
@@ -37,17 +38,15 @@
 namespace arrow {
 namespace fs {
 
+using ::arrow::internal::IOErrorFromErrno;
+#ifdef _WIN32
+using ::arrow::internal::IOErrorFromWinError;
+#endif
 using ::arrow::internal::NativePathString;
 using ::arrow::internal::PlatformFilename;
 
 namespace {
 
-template <typename... Args>
-Status ErrnoToStatus(Args&&... args) {
-  auto err_string = ::arrow::internal::ErrnoMessage(errno);
-  return Status::IOError(std::forward<Args>(args)..., err_string);
-}
-
 #ifdef _WIN32
 
 std::string NativeToString(const NativePathString& ns) {
@@ -55,12 +54,6 @@ std::string NativeToString(const NativePathString& ns) {
   return fn.ToString();
 }
 
-template <typename... Args>
-Status WinErrorToStatus(Args&&... args) {
-  auto err_string = ::arrow::internal::WinErrorMessage(GetLastError());
-  return Status::IOError(std::forward<Args>(args)..., err_string);
-}
-
 TimePoint ToTimePoint(FILETIME ft) {
   // Hundreds of nanoseconds between January 1, 1601 (UTC) and the Unix epoch.
   static constexpr int64_t kFileTimeEpoch = 11644473600LL * 10000000;
@@ -107,13 +100,15 @@ Result<FileStats> StatFile(const std::wstring& path) {
       st.set_size(kNoSize);
       return st;
     } else {
-      return WinErrorToStatus("Failed querying information for path '", bytes_path, "'");
+      return IOErrorFromWinError(GetLastError(), "Failed querying information for path '",
+                                 bytes_path, "'");
     }
   }
   BY_HANDLE_FILE_INFORMATION info;
   if (!GetFileInformationByHandle(h, &info)) {
     CloseHandle(h);
-    return WinErrorToStatus("Failed querying information for path '", bytes_path, "'");
+    return IOErrorFromWinError(GetLastError(), "Failed querying information for path '",
+                               bytes_path, "'");
   }
   CloseHandle(h);
   st = FileInformationToFileStat(info);
@@ -160,7 +155,7 @@ Result<FileStats> StatFile(const std::string& path) {
       st.set_mtime(kNoTime);
       st.set_size(kNoSize);
     } else {
-      return ErrnoToStatus("Failed stat()ing path '", path, "'");
+      return IOErrorFromErrno(errno, "Failed stat()ing path '", path, "'");
     }
   } else {
     st = StatToFileStat(s);
@@ -236,32 +231,31 @@ Status LocalFileSystem::CreateDir(const std::string& path, bool recursive) {
 
 Status LocalFileSystem::DeleteDir(const std::string& path) {
   ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
-  ARROW_ASSIGN_OR_RAISE(bool deleted, ::arrow::internal::DeleteDirTree(fn));
-  if (deleted) {
-    return Status::OK();
-  } else {
-    return Status::IOError("Directory does not exist: '", path, "'");
+  auto st = ::arrow::internal::DeleteDirTree(fn, /*allow_non_existent=*/false).status();
+  if (!st.ok()) {
+    // TODO Status::WithPrefix()?
+    std::stringstream ss;
+    ss << "Cannot delete directory '" << path << "': " << st.message();
+    return st.WithMessage(ss.str());
   }
+  return Status::OK();
 }
 
 Status LocalFileSystem::DeleteDirContents(const std::string& path) {
   ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
-  ARROW_ASSIGN_OR_RAISE(bool deleted, ::arrow::internal::DeleteDirContents(fn));
-  if (deleted) {
-    return Status::OK();
-  } else {
-    return Status::IOError("Directory does not exist: '", path, "'");
+  auto st =
+      ::arrow::internal::DeleteDirContents(fn, /*allow_non_existent=*/false).status();
+  if (!st.ok()) {
+    std::stringstream ss;
+    ss << "Cannot delete directory contents in '" << path << "': " << st.message();
+    return st.WithMessage(ss.str());
   }
+  return Status::OK();
 }
 
 Status LocalFileSystem::DeleteFile(const std::string& path) {
   ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
-  ARROW_ASSIGN_OR_RAISE(bool deleted, arrow::internal::DeleteFile(fn));
-  if (deleted) {
-    return Status::OK();
-  } else {
-    return Status::IOError("File does not exist: '", path, "'");
-  }
+  return ::arrow::internal::DeleteFile(fn, /*allow_non_existent=*/false).status();
 }
 
 Status LocalFileSystem::Move(const std::string& src, const std::string& dest) {
@@ -271,13 +265,13 @@ Status LocalFileSystem::Move(const std::string& src, const std::string& dest) {
 #ifdef _WIN32
   if (!MoveFileExW(sfn.ToNative().c_str(), dfn.ToNative().c_str(),
                    MOVEFILE_REPLACE_EXISTING)) {
-    return WinErrorToStatus("Failed renaming '", sfn.ToString(), "' to '", dfn.ToString(),
-                            "': ");
+    return IOErrorFromWinError(GetLastError(), "Failed renaming '", sfn.ToString(),
+                               "' to '", dfn.ToString(), "'");
   }
 #else
   if (rename(sfn.ToNative().c_str(), dfn.ToNative().c_str()) == -1) {
-    return ErrnoToStatus("Failed renaming '", sfn.ToString(), "' to '", dfn.ToString(),
-                         "': ");
+    return IOErrorFromErrno(errno, "Failed renaming '", sfn.ToString(), "' to '",
+                            dfn.ToString(), "'");
   }
 #endif
   return Status::OK();
@@ -294,8 +288,8 @@ Status LocalFileSystem::CopyFile(const std::string& src, const std::string& dest
 #ifdef _WIN32
   if (!CopyFileW(sfn.ToNative().c_str(), dfn.ToNative().c_str(),
                  FALSE /* bFailIfExists */)) {
-    return WinErrorToStatus("Failed copying '", sfn.ToString(), "' to '", dfn.ToString(),
-                            "': ");
+    return IOErrorFromWinError(GetLastError(), "Failed copying '", sfn.ToString(),
+                               "' to '", dfn.ToString(), "'");
   }
   return Status::OK();
 #else

cpp/src/arrow/filesystem/localfs_test.cc

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <cerrno>
 #include <chrono>
 #include <memory>
 #include <string>

cpp/src/arrow/io/file.cc

@@ -593,7 +593,8 @@ class MemoryMappedFile::MemoryMap
     void* result = mmap(nullptr, mmap_length, prot_flags_, map_mode_, file_->fd(),
                         static_cast<off_t>(offset));
     if (result == MAP_FAILED) {
-      return Status::IOError("Memory mapping file failed: ", std::strerror(errno));
+      return Status::IOError("Memory mapping file failed: ",
+                             ::arrow::internal::ErrnoMessage(errno));
     }
     map_len_ = mmap_length;
     offset_ = offset;

cpp/src/arrow/status.h

@@ -161,101 +161,103 @@ class ARROW_EXPORT Status : public util::EqualityComparable<Status>,
   /// Return a success status
   static Status OK() { return Status(); }
 
+  template <typename... Args>
+  static Status FromArgs(StatusCode code, Args&&... args) {
+    return Status(code, util::StringBuilder(std::forward<Args>(args)...));
+  }
+
+  template <typename... Args>
+  static Status FromDetailAndArgs(StatusCode code, std::shared_ptr<StatusDetail> detail,
+                                  Args&&... args) {
+    return Status(code, util::StringBuilder(std::forward<Args>(args)...),
+                  std::move(detail));
+  }
+
   /// Return an error status for out-of-memory conditions
   template <typename... Args>
   static Status OutOfMemory(Args&&... args) {
-    return Status(StatusCode::OutOfMemory,
-                  util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::OutOfMemory, std::forward<Args>(args)...);
   }
 
   /// Return an error status for failed key lookups (e.g. column name in a table)
   template <typename... Args>
   static Status KeyError(Args&&... args) {
-    return Status(StatusCode::KeyError, util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::KeyError, std::forward<Args>(args)...);
   }
 
   /// Return an error status for type errors (such as mismatching data types)
   template <typename... Args>
   static Status TypeError(Args&&... args) {
-    return Status(StatusCode::TypeError,
-                  util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::TypeError, std::forward<Args>(args)...);
   }
 
   /// Return an error status for unknown errors
   template <typename... Args>
   static Status UnknownError(Args&&... args) {
-    return Status(StatusCode::UnknownError,
-                  util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::UnknownError, std::forward<Args>(args)...);
   }
 
   /// Return an error status when an operation or a combination of operation and
   /// data types is unimplemented
   template <typename... Args>
   static Status NotImplemented(Args&&... args) {
-    return Status(StatusCode::NotImplemented,
-                  util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::NotImplemented, std::forward<Args>(args)...);
   }
 
   /// Return an error status for invalid data (for example a string that fails parsing)
   template <typename... Args>
   static Status Invalid(Args&&... args) {
-    return Status(StatusCode::Invalid, util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::Invalid, std::forward<Args>(args)...);
   }
 
   /// Return an error status when an index is out of bounds
   template <typename... Args>
   static Status IndexError(Args&&... args) {
-    return Status(StatusCode::IndexError,
-                  util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::IndexError, std::forward<Args>(args)...);
   }
 
   /// Return an error status when a container's capacity would exceed its limits
   template <typename... Args>
   static Status CapacityError(Args&&... args) {
-    return Status(StatusCode::CapacityError,
-                  util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::CapacityError, std::forward<Args>(args)...);
   }
 
   /// Return an error status when some IO-related operation failed
   template <typename... Args>
   static Status IOError(Args&&... args) {
-    return Status(StatusCode::IOError, util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::IOError, std::forward<Args>(args)...);
   }
 
   /// Return an error status when some (de)serialization operation failed
   template <typename... Args>
   static Status SerializationError(Args&&... args) {
-    return Status(StatusCode::SerializationError,
-                  util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::SerializationError, std::forward<Args>(args)...);
   }
 
   template <typename... Args>
   static Status RError(Args&&... args) {
-    return Status(StatusCode::RError, util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::RError, std::forward<Args>(args)...);
   }
 
   template <typename... Args>
   static Status CodeGenError(Args&&... args) {
-    return Status(StatusCode::CodeGenError,
-                  util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::CodeGenError, std::forward<Args>(args)...);
   }
 
   template <typename... Args>
   static Status ExpressionValidationError(Args&&... args) {
-    return Status(StatusCode::ExpressionValidationError,
-                  util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::ExpressionValidationError,
+                            std::forward<Args>(args)...);
   }
 
   template <typename... Args>
   static Status ExecutionError(Args&&... args) {
-    return Status(StatusCode::ExecutionError,
-                  util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::ExecutionError, std::forward<Args>(args)...);
   }
 
   template <typename... Args>
   static Status AlreadyExists(Args&&... args) {
-    return Status(StatusCode::AlreadyExists,
-                  util::StringBuilder(std::forward<Args>(args)...));
+    return Status::FromArgs(StatusCode::AlreadyExists, std::forward<Args>(args)...);
   }
 
   /// Return true iff the status indicates success.

cpp/src/arrow/util/io_util.cc

@@ -86,6 +86,7 @@
 
 #include "arrow/buffer.h"
 #include "arrow/result.h"
+#include "arrow/util/checked_cast.h"
 #include "arrow/util/io_util.h"
 #include "arrow/util/logging.h"
 
@@ -95,6 +96,9 @@
 #endif
 
 namespace arrow {
+
+using internal::checked_cast;
+
 namespace io {
 
 //
@@ -266,6 +270,80 @@ std::string WinErrorMessage(int errnum) {
 }
 #endif
 
+namespace {
+
+const char kErrnoDetailTypeId[] = "arrow::ErrnoDetail";
+
+class ErrnoDetail : public StatusDetail {
+ public:
+  explicit ErrnoDetail(int errnum) : errnum_(errnum) {}
+
+  const char* type_id() const override { return kErrnoDetailTypeId; }
+
+  std::string ToString() const override {
+    std::stringstream ss;
+    ss << "[errno " << errnum_ << "] " << ErrnoMessage(errnum_);
+    return ss.str();
+  }
+
+  int errnum() const { return errnum_; }
+
+ protected:
+  int errnum_;
+};
+
+#if _WIN32
+const char kWinErrorDetailTypeId[] = "arrow::WinErrorDetail";
+
+class WinErrorDetail : public StatusDetail {
+ public:
+  explicit WinErrorDetail(int errnum) : errnum_(errnum) {}
+
+  const char* type_id() const override { return kWinErrorDetailTypeId; }
+
+  std::string ToString() const override {
+    std::stringstream ss;
+    ss << "[Windows error " << errnum_ << "] " << WinErrorMessage(errnum_);
+    return ss.str();
+  }
+
+  int errnum() const { return errnum_; }
+
+ protected:
+  int errnum_;
+};
+#endif
+
+}  // namespace
+
+std::shared_ptr<StatusDetail> StatusDetailFromErrno(int errnum) {
+  return std::make_shared<ErrnoDetail>(errnum);
+}
+
+#if _WIN32
+std::shared_ptr<StatusDetail> StatusDetailFromWinError(int errnum) {
+  return std::make_shared<WinErrorDetail>(errnum);
+}
+#endif
+
+int ErrnoFromStatus(const Status& status) {
+  const auto detail = status.detail();
+  if (detail != nullptr && detail->type_id() == kErrnoDetailTypeId) {
+    return checked_cast<const ErrnoDetail&>(*detail).errnum();
+  }
+  return 0;
+}
+
+int WinErrorFromStatus(const Status& status) {
+#if _WIN32
+  const auto detail = status.detail();
+  if (detail != nullptr && detail->type_id() == kWinErrorDetailTypeId) {
+    return checked_cast<const WinErrorDetail&>(*detail).errnum();
+  }
+#endif
+  return 0;
+}
+
 //
 // PlatformFilename implementation
 //
@@ -377,8 +455,8 @@ Result<bool> DoCreateDir(const PlatformFilename& dir_path, bool create_parents)
       return DoCreateDir(dir_path, false);  // Retry
     }
   }
-  return Status::IOError("Cannot create directory '", dir_path.ToString(),
-                         "': ", WinErrorMessage(errnum));
+  return IOErrorFromWinError(GetLastError(), "Cannot create directory '",
+                             dir_path.ToString(), "'");
 #else
   if (mkdir(dir_path.ToNative().c_str(), S_IRWXU | S_IRWXG | S_IRWXO) == 0) {
     return true;
@@ -393,8 +471,7 @@ Result<bool> DoCreateDir(const PlatformFilename& dir_path, bool create_parents)
       return DoCreateDir(dir_path, false);  // Retry
     }
   }
-  return Status::IOError("Cannot create directory '", dir_path.ToString(),
-                         "': ", ErrnoMessage(errno));
+  return IOErrorFromErrno(errno, "Cannot create directory '", dir_path.ToString(), "'");
 #endif
 }
 
@@ -432,8 +509,8 @@ Result<std::vector<WIN32_FIND_DATAW>> ListDirInternal(const PlatformFilename& di
   std::wstring pattern = PathWithoutTrailingSlash(dir_path) + L"\\*.*";
   HANDLE handle = FindFirstFileW(pattern.c_str(), &find_data);
   if (handle == INVALID_HANDLE_VALUE) {
-    return Status::IOError("Cannot list directory '", dir_path.ToString(),
-                           "': ", WinErrorMessage(GetLastError()));
+    return IOErrorFromWinError(GetLastError(), "Cannot list directory '",
+                               dir_path.ToString(), "'");
   }
 
   std::unique_ptr<HANDLE, decltype(&FindHandleDeleter)> handle_guard(&handle,
@@ -453,8 +530,8 @@ Result<std::vector<WIN32_FIND_DATAW>> ListDirInternal(const PlatformFilename& di
 
   int errnum = GetLastError();
   if (errnum != ERROR_NO_MORE_FILES) {
-    return Status::IOError("Cannot list directory '", dir_path.ToString(),
-                           "': ", WinErrorMessage(errnum));
+    return IOErrorFromWinError(GetLastError(), "Cannot list directory '",
+                               dir_path.ToString(), "'");
   }
   return results;
 }
@@ -466,8 +543,8 @@ Status FindOneFile(const PlatformFilename& fn, WIN32_FIND_DATAW* find_data,
     int errnum = GetLastError();
     if (exists == nullptr ||
         (errnum != ERROR_PATH_NOT_FOUND && errnum != ERROR_FILE_NOT_FOUND)) {
-      return Status::IOError("Cannot get information for path '", fn.ToString(),
-                             "': ", WinErrorMessage(errnum));
+      return IOErrorFromWinError(GetLastError(), "Cannot get information for path '",
+                                 fn.ToString(), "'");
     }
     *exists = false;
   } else {
@@ -497,8 +574,7 @@ Result<std::vector<PlatformFilename>> ListDir(const PlatformFilename& dir_path)
 Result<std::vector<PlatformFilename>> ListDir(const PlatformFilename& dir_path) {
   DIR* dir = opendir(dir_path.ToNative().c_str());
   if (dir == nullptr) {
-    return Status::IOError("Cannot list directory '", dir_path.ToString(),
-                           "': ", ErrnoMessage(errno));
+    return IOErrorFromErrno(errno, "Cannot list directory '", dir_path.ToString(), "'");
   }
 
   auto dir_deleter = [](DIR* dir) -> void {
@@ -519,8 +595,7 @@ Result<std::vector<PlatformFilename>> ListDir(const PlatformFilename& dir_path)
     entry = readdir(dir);
   }
   if (errno != 0) {
-    return Status::IOError("Cannot list directory '", dir_path.ToString(),
-                           "': ", ErrnoMessage(errno));
+    return IOErrorFromErrno(errno, "Cannot list directory '", dir_path.ToString(), "'");
   }
   return results;
 }
@@ -543,8 +618,8 @@ Status DeleteDirEntryDir(const PlatformFilename& path, const WIN32_FIND_DATAW& e
   if (remove_top_dir) {
     // Remove now empty directory or reparse point (e.g. symlink to dir)
     if (!RemoveDirectoryW(path.ToNative().c_str())) {
-      return Status::IOError("Cannot delete directory entry '", path.ToString(),
-                             "': ", WinErrorMessage(GetLastError()));
+      return IOErrorFromWinError(GetLastError(), "Cannot delete directory entry '",
+                                 path.ToString(), "': ");
     }
   }
   return Status::OK();
@@ -556,8 +631,8 @@ Status DeleteDirEntry(const PlatformFilename& path, const WIN32_FIND_DATAW& entr
   }
   // It's a non-directory entry, most likely a regular file
   if (!DeleteFileW(path.ToNative().c_str())) {
-    return Status::IOError("Cannot delete file '", path.ToString(),
-                           "': ", WinErrorMessage(GetLastError()));
+    return IOErrorFromWinError(GetLastError(), "Cannot delete file '", path.ToString(),
+                               "': ");
   }
   return Status::OK();
 }
@@ -571,10 +646,16 @@ Status DeleteDirTreeInternal(const PlatformFilename& dir_path) {
   return Status::OK();
 }
 
-Result<bool> DeleteDirContents(const PlatformFilename& dir_path, bool remove_top_dir) {
-  bool exists = false;
+Result<bool> DeleteDirContents(const PlatformFilename& dir_path, bool allow_non_existent,
+                               bool remove_top_dir) {
+  bool exists = true;
   WIN32_FIND_DATAW entry;
-  RETURN_NOT_OK(FindOneFile(dir_path, &entry, &exists));
+  if (allow_non_existent) {
+    RETURN_NOT_OK(FindOneFile(dir_path, &entry, &exists));
+  } else {
+    // Will raise if dir_path does not exist
+    RETURN_NOT_OK(FindOneFile(dir_path, &entry));
+  }
   if (exists) {
     RETURN_NOT_OK(DeleteDirEntryDir(dir_path, entry, remove_top_dir));
   }
@@ -586,8 +667,8 @@ Result<bool> DeleteDirContents(const PlatformFilename& dir_path, bool remove_top
 Status LinkStat(const PlatformFilename& path, struct stat* lst, bool* exists = nullptr) {
   if (lstat(path.ToNative().c_str(), lst) != 0) {
     if (exists == nullptr || (errno != ENOENT && errno != ENOTDIR && errno != ELOOP)) {
-      return Status::IOError("Cannot get information for path '", path.ToString(),
-                             "': ", ErrnoMessage(errno));
+      return IOErrorFromErrno(errno, "Cannot get information for path '", path.ToString(),
+                              "'");
     }
     *exists = false;
   } else if (exists != nullptr) {
@@ -605,14 +686,14 @@ Status DeleteDirEntryDir(const PlatformFilename& path, const struct stat& lst,
     DCHECK(S_ISDIR(lst.st_mode));
     RETURN_NOT_OK(DeleteDirTreeInternal(path));
     if (remove_top_dir && rmdir(path.ToNative().c_str()) != 0) {
-      return Status::IOError("Cannot delete directory entry '", path.ToString(),
-                             "': ", ErrnoMessage(errno));
+      return IOErrorFromErrno(errno, "Cannot delete directory entry '", path.ToString(),
+                              "'");
     }
   } else {
     // Remove symlink
     if (remove_top_dir && unlink(path.ToNative().c_str()) != 0) {
-      return Status::IOError("Cannot delete directory entry '", path.ToString(),
-                             "': ", ErrnoMessage(errno));
+      return IOErrorFromErrno(errno, "Cannot delete directory entry '", path.ToString(),
+                              "'");
     }
   }
   return Status::OK();
@@ -623,8 +704,8 @@ Status DeleteDirEntry(const PlatformFilename& path, const struct stat& lst) {
     return DeleteDirEntryDir(path, lst);
   }
   if (unlink(path.ToNative().c_str()) != 0) {
-    return Status::IOError("Cannot delete directory entry '", path.ToString(),
-                           "': ", ErrnoMessage(errno));
+    return IOErrorFromErrno(errno, "Cannot delete directory entry '", path.ToString(),
+                            "'");
   }
   return Status::OK();
 }
@@ -640,10 +721,16 @@ Status DeleteDirTreeInternal(const PlatformFilename& dir_path) {
   return Status::OK();
 }
 
-Result<bool> DeleteDirContents(const PlatformFilename& dir_path, bool remove_top_dir) {
+Result<bool> DeleteDirContents(const PlatformFilename& dir_path, bool allow_non_existent,
+                               bool remove_top_dir) {
   bool exists = true;
   struct stat lst;
-  RETURN_NOT_OK(LinkStat(dir_path, &lst, &exists));
+  if (allow_non_existent) {
+    RETURN_NOT_OK(LinkStat(dir_path, &lst, &exists));
+  } else {
+    // Will raise if dir_path does not exist
+    RETURN_NOT_OK(LinkStat(dir_path, &lst));
+  }
   if (exists) {
     if (!S_ISDIR(lst.st_mode) && !S_ISLNK(lst.st_mode)) {
       return Status::IOError("Cannot delete directory '", dir_path.ToString(),
@@ -658,32 +745,32 @@ Result<bool> DeleteDirContents(const PlatformFilename& dir_path, bool remove_top
 
 }  // namespace
 
-Result<bool> DeleteDirContents(const PlatformFilename& dir_path) {
-  return DeleteDirContents(dir_path, /*remove_top_dir=*/false);
+Result<bool> DeleteDirContents(const PlatformFilename& dir_path,
+                               bool allow_non_existent) {
+  return DeleteDirContents(dir_path, allow_non_existent, /*remove_top_dir=*/false);
 }
 
-Result<bool> DeleteDirTree(const PlatformFilename& dir_path) {
-  return DeleteDirContents(dir_path, /*remove_top_dir=*/true);
+Result<bool> DeleteDirTree(const PlatformFilename& dir_path, bool allow_non_existent) {
+  return DeleteDirContents(dir_path, allow_non_existent, /*remove_top_dir=*/true);
 }
 
-Result<bool> DeleteFile(const PlatformFilename& file_path) {
+Result<bool> DeleteFile(const PlatformFilename& file_path, bool allow_non_existent) {
 #ifdef _WIN32
   if (DeleteFileW(file_path.ToNative().c_str())) {
     return true;
   } else {
     int errnum = GetLastError();
-    if (errnum != ERROR_FILE_NOT_FOUND) {
-      return Status::IOError("Cannot delete file '", file_path.ToString(),
-                             "': ", WinErrorMessage(errnum));
+    if (!allow_non_existent || errnum != ERROR_FILE_NOT_FOUND) {
+      return IOErrorFromWinError(GetLastError(), "Cannot delete file '",
+                                 file_path.ToString(), "'");
     }
   }
 #else
   if (unlink(file_path.ToNative().c_str()) == 0) {
     return true;
   } else {
-    if (errno != ENOENT) {
-      return Status::IOError("Cannot delete file '", file_path.ToString(),
-                             "': ", ErrnoMessage(errno));
+    if (!allow_non_existent || errno != ENOENT) {
+      return IOErrorFromErrno(errno, "Cannot delete file '", file_path.ToString(), "'");
     }
   }
 #endif
@@ -697,8 +784,8 @@ Result<bool> FileExists(const PlatformFilename& path) {
   } else {
     int errnum = GetLastError();
     if (errnum != ERROR_PATH_NOT_FOUND && errnum != ERROR_FILE_NOT_FOUND) {
-      return Status::IOError("Failed getting information for path '", path.ToString(),
-                             "': ", WinErrorMessage(errnum));
+      return IOErrorFromWinError(GetLastError(), "Failed getting information for path '",
+                                 path.ToString(), "'");
     }
     return false;
   }
@@ -708,8 +795,8 @@ Result<bool> FileExists(const PlatformFilename& path) {
     return true;
   } else {
     if (errno != ENOENT && errno != ENOTDIR) {
-      return Status::IOError("Failed getting information for path '", path.ToString(),
-                             "': ", ErrnoMessage(errno));
+      return IOErrorFromErrno(errno, "Failed getting information for path '",
+                              path.ToString(), "'");
     }
     return false;
   }
@@ -738,12 +825,12 @@ static inline Result<int> CheckFileOpResult(int fd_ret, int errno_actual,
 #ifdef _WIN32
     int winerr = GetLastError();
     if (winerr != ERROR_SUCCESS) {
-      return Status::IOError("Failed to ", opname, " file '", file_name.ToString(),
-                             "', error: ", WinErrorMessage(winerr));
+      return IOErrorFromWinError(GetLastError(), "Failed to ", opname, " file '",
+                                 file_name.ToString(), "'");
     }
 #endif
-    return Status::IOError("Failed to ", opname, " file '", file_name.ToString(),
-                           "', error: ", ErrnoMessage(errno_actual));
+    return IOErrorFromErrno(errno_actual, "Failed to ", opname, " file '",
+                            file_name.ToString(), "'");
   }
   return fd_ret;
 }
@@ -856,16 +943,16 @@ Result<Pipe> CreatePipe() {
 #endif
 
   if (ret == -1) {
-    return Status::IOError("Error creating pipe: ", ErrnoMessage(errno));
+    return IOErrorFromErrno(errno, "Error creating pipe");
   }
   return Pipe{fd[0], fd[1]};
 }
 
-static Status StatusFromErrno(const char* prefix) {
+static Status StatusFromMmapErrno(const char* prefix) {
 #ifdef _WIN32
   errno = __map_mman_error(GetLastError(), EPERM);
 #endif
-  return Status::IOError(prefix, ErrnoMessage(errno));
+  return IOErrorFromErrno(errno, prefix);
 }
 
 //
@@ -881,12 +968,12 @@ Status MemoryMapRemap(void* addr, size_t old_size, size_t new_size, int fildes,
   HANDLE fm, h;
 
   if (!UnmapViewOfFile(addr)) {
-    return StatusFromErrno("UnmapViewOfFile failed: ");
+    return StatusFromMmapErrno("UnmapViewOfFile failed");
   }
 
   h = reinterpret_cast<HANDLE>(_get_osfhandle(fildes));
   if (h == INVALID_HANDLE_VALUE) {
-    return StatusFromErrno("Cannot get file handle: ");
+    return StatusFromMmapErrno("Cannot get file handle");
   }
 
   uint64_t new_size64 = new_size;
@@ -897,12 +984,12 @@ Status MemoryMapRemap(void* addr, size_t old_size, size_t new_size, int fildes,
   SetEndOfFile(h);
   fm = CreateFileMapping(h, NULL, PAGE_READWRITE, 0, 0, "");
   if (fm == NULL) {
-    return StatusFromErrno("CreateFileMapping failed: ");
+    return StatusFromMmapErrno("CreateFileMapping failed");
   }
   *new_addr = MapViewOfFile(fm, FILE_MAP_WRITE, 0, 0, new_size);
   CloseHandle(fm);
   if (new_addr == NULL) {
-    return StatusFromErrno("MapViewOfFile failed: ");
+    return StatusFromMmapErrno("MapViewOfFile failed");
   }
   return Status::OK();
 #else
@@ -910,25 +997,25 @@ Status MemoryMapRemap(void* addr, size_t old_size, size_t new_size, int fildes,
   // we have to close the mmap first, truncate the file to the new size
   // and recreate the mmap
   if (munmap(addr, old_size) == -1) {
-    return StatusFromErrno("munmap failed: ");
+    return StatusFromMmapErrno("munmap failed");
   }
   if (ftruncate(fildes, new_size) == -1) {
-    return StatusFromErrno("ftruncate failed: ");
+    return StatusFromMmapErrno("ftruncate failed");
   }
   // we set READ / WRITE flags on the new map, since we could only have
   // unlarged a RW map in the first place
   *new_addr = mmap(NULL, new_size, PROT_READ | PROT_WRITE, MAP_SHARED, fildes, 0);
   if (*new_addr == MAP_FAILED) {
-    return StatusFromErrno("mmap failed: ");
+    return StatusFromMmapErrno("mmap failed");
   }
   return Status::OK();
 #else
   if (ftruncate(fildes, new_size) == -1) {
-    return StatusFromErrno("ftruncate failed: ");
+    return StatusFromMmapErrno("ftruncate failed");
   }
   *new_addr = mremap(addr, old_size, new_size, MREMAP_MAYMOVE);
   if (*new_addr == MAP_FAILED) {
-    return StatusFromErrno("mremap failed: ");
+    return StatusFromMmapErrno("mremap failed");
   }
   return Status::OK();
 #endif
@@ -1034,7 +1121,7 @@ Result<int64_t> FileRead(int fd, uint8_t* buffer, int64_t nbytes) {
 #endif
 
     if (ret == -1) {
-      return Status::IOError("Error reading bytes from file: ", ErrnoMessage(errno));
+      return IOErrorFromErrno(errno, "Error reading bytes from file");
     }
     if (ret == 0) {
       // EOF
@@ -1055,7 +1142,7 @@ Result<int64_t> FileReadAt(int fd, uint8_t* buffer, int64_t position, int64_t nb
     int64_t ret = pread_compat(fd, buffer, chunksize, position);
 
     if (ret == -1) {
-      return Status::IOError("Error reading bytes from file: ", ErrnoMessage(errno));
+      return IOErrorFromErrno(errno, "Error reading bytes from file");
     }
     if (ret == 0) {
       // EOF
@@ -1093,7 +1180,7 @@ Status FileWrite(int fd, const uint8_t* buffer, const int64_t nbytes) {
   }
 
   if (ret == -1) {
-    return Status::IOError("Error writing bytes to file: ", ErrnoMessage(errno));
+    return IOErrorFromErrno(errno, "Error writing bytes to file");
   }
   return Status::OK();
 }
@@ -1110,7 +1197,7 @@ Status FileTruncate(int fd, const int64_t size) {
 #endif
 
   if (ret == -1) {
-    return Status::IOError("Error writing bytes to file: ", ErrnoMessage(errno_actual));
+    return IOErrorFromErrno(errno_actual, "Error writing bytes to file");
   }
   return Status::OK();
 }

cpp/src/arrow/util/io_util.h

@@ -165,13 +165,15 @@ Result<bool> CreateDirTree(const PlatformFilename& dir_path);
 ///
 /// Return whether the directory existed.
 ARROW_EXPORT
-Result<bool> DeleteDirContents(const PlatformFilename& dir_path);
+Result<bool> DeleteDirContents(const PlatformFilename& dir_path,
+                               bool allow_non_existent = true);
 
 /// Delete a directory tree if it exists.
 ///
 /// Return whether the directory existed.
 ARROW_EXPORT
-Result<bool> DeleteDirTree(const PlatformFilename& dir_path);
+Result<bool> DeleteDirTree(const PlatformFilename& dir_path,
+                           bool allow_non_existent = true);
 
 // Non-recursively list the contents of the given directory.
 // The returned names are the children's base names, not including dir_path.
@@ -182,7 +184,8 @@ Result<std::vector<PlatformFilename>> ListDir(const PlatformFilename& dir_path);
 ///
 /// Return whether the file existed.
 ARROW_EXPORT
-Result<bool> DeleteFile(const PlatformFilename& file_path);
+Result<bool> DeleteFile(const PlatformFilename& file_path,
+                        bool allow_non_existent = true);
 
 /// Return whether a file exists.
 ARROW_EXPORT
@@ -258,6 +261,44 @@ ARROW_EXPORT
 std::string WinErrorMessage(int errnum);
 #endif
 
+ARROW_EXPORT
+std::shared_ptr<StatusDetail> StatusDetailFromErrno(int errnum);
+#if _WIN32
+ARROW_EXPORT
+std::shared_ptr<StatusDetail> StatusDetailFromWinError(int errnum);
+#endif
+
+template <typename... Args>
+Status StatusFromErrno(int errnum, StatusCode code, Args&&... args) {
+  return Status::FromDetailAndArgs(code, StatusDetailFromErrno(errnum),
+                                   std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+Status IOErrorFromErrno(int errnum, Args&&... args) {
+  return StatusFromErrno(errnum, StatusCode::IOError, std::forward<Args>(args)...);
+}
+
+#if _WIN32
+template <typename... Args>
+Status StatusFromWinError(int errnum, StatusCode code, Args&&... args) {
+  return Status::FromDetailAndArgs(code, StatusDetailFromWinError(errnum),
+                                   std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+Status IOErrorFromWinError(int errnum, Args&&... args) {
+  return StatusFromWinError(errnum, StatusCode::IOError, std::forward<Args>(args)...);
+}
+#endif
+
+ARROW_EXPORT
+int ErrnoFromStatus(const Status&);
+
+// Always returns 0 on non-Windows platforms (for Python).
+ARROW_EXPORT
+int WinErrorFromStatus(const Status&);
+
 class ARROW_EXPORT TemporaryDir {
  public:
   ~TemporaryDir();

cpp/src/arrow/util/io_util_test.cc

@@ -16,12 +16,20 @@
 // under the License.
 
 #include <algorithm>
+#include <cerrno>
 #include <vector>
 
 #include <gtest/gtest.h>
 
 #include "arrow/testing/gtest_util.h"
 #include "arrow/util/io_util.h"
+#include "arrow/util/windows_compatibility.h"
+
+#ifdef _WIN32
+#ifdef DeleteFile
+#undef DeleteFile
+#endif
+#endif
 
 namespace arrow {
 namespace internal {
@@ -38,6 +46,40 @@ void AssertNotExists(const PlatformFilename& path) {
   ASSERT_FALSE(exists) << "Path '" << path.ToString() << "' exists";
 }
 
+TEST(ErrnoFromStatus, Basics) {
+  Status st;
+  st = Status::OK();
+  ASSERT_EQ(ErrnoFromStatus(st), 0);
+  st = Status::KeyError("foo");
+  ASSERT_EQ(ErrnoFromStatus(st), 0);
+  st = Status::IOError("foo");
+  ASSERT_EQ(ErrnoFromStatus(st), 0);
+  st = StatusFromErrno(EINVAL, StatusCode::KeyError, "foo");
+  ASSERT_EQ(ErrnoFromStatus(st), EINVAL);
+  st = IOErrorFromErrno(EPERM, "foo");
+  ASSERT_EQ(ErrnoFromStatus(st), EPERM);
+  st = IOErrorFromErrno(6789, "foo");
+  ASSERT_EQ(ErrnoFromStatus(st), 6789);
+}
+
+#if _WIN32
+TEST(WinErrorFromStatus, Basics) {
+  Status st;
+  st = Status::OK();
+  ASSERT_EQ(WinErrorFromStatus(st), 0);
+  st = Status::KeyError("foo");
+  ASSERT_EQ(WinErrorFromStatus(st), 0);
+  st = Status::IOError("foo");
+  ASSERT_EQ(WinErrorFromStatus(st), 0);
+  st = StatusFromWinError(ERROR_FILE_NOT_FOUND, StatusCode::KeyError, "foo");
+  ASSERT_EQ(WinErrorFromStatus(st), ERROR_FILE_NOT_FOUND);
+  st = IOErrorFromWinError(ERROR_ACCESS_DENIED, "foo");
+  ASSERT_EQ(WinErrorFromStatus(st), ERROR_ACCESS_DENIED);
+  st = IOErrorFromWinError(6789, "foo");
+  ASSERT_EQ(WinErrorFromStatus(st), 6789);
+}
+#endif
+
 TEST(PlatformFilename, RoundtripAscii) {
   PlatformFilename fn;
   ASSERT_OK_AND_ASSIGN(fn, PlatformFilename::FromString("a/b"));
@@ -297,6 +339,14 @@ TEST(CreateDirDeleteDir, Basics) {
   // It's not an error to call DeleteDirTree on a non-existent path.
   ASSERT_OK_AND_ASSIGN(deleted, DeleteDirTree(parent));
   ASSERT_FALSE(deleted);
+  // ... unless asked so
+  auto status = DeleteDirTree(parent, /*allow_non_existent=*/false).status();
+  ASSERT_RAISES(IOError, status);
+#ifdef _WIN32
+  ASSERT_EQ(WinErrorFromStatus(status), ERROR_FILE_NOT_FOUND);
+#else
+  ASSERT_EQ(ErrnoFromStatus(status), ENOENT);
+#endif
 }
 
 TEST(DeleteDirContents, Basics) {
@@ -341,6 +391,14 @@ TEST(DeleteDirContents, Basics) {
   // It's not an error to call DeleteDirContents on a non-existent path.
   ASSERT_OK_AND_ASSIGN(deleted, DeleteDirContents(child1));
   ASSERT_FALSE(deleted);
+  // ... unless asked so
+  auto status = DeleteDirContents(child1, /*allow_non_existent=*/false).status();
+  ASSERT_RAISES(IOError, status);
+#ifdef _WIN32
+  ASSERT_EQ(WinErrorFromStatus(status), ERROR_FILE_NOT_FOUND);
+#else
+  ASSERT_EQ(ErrnoFromStatus(status), ENOENT);
+#endif
 }
 
 TEST(TemporaryDir, Basics) {
@@ -455,6 +513,13 @@ TEST(DeleteFile, Basics) {
   ASSERT_OK_AND_ASSIGN(deleted, DeleteFile(fn));
   ASSERT_FALSE(deleted);
   AssertNotExists(fn);
+  auto status = DeleteFile(fn, /*allow_non_existent=*/false).status();
+  ASSERT_RAISES(IOError, status);
+#ifdef _WIN32
+  ASSERT_EQ(WinErrorFromStatus(status), ERROR_FILE_NOT_FOUND);
+#else
+  ASSERT_EQ(ErrnoFromStatus(status), ENOENT);
+#endif
 
   // Cannot call DeleteFile on directory
   ASSERT_OK_AND_ASSIGN(fn, temp_dir->path().Join("test-temp_dir"));

python/pyarrow/_fs.pyx

@@ -253,7 +253,7 @@ cdef class FileSystem:
 
         return [FileStats.wrap(stat) for stat in stats]
 
-    def create_dir(self, path, bint recursive=True):
+    def create_dir(self, path, *, bint recursive=True):
         """Create a directory and subdirectories.
 
         This function succeeds if the directory already exists.

python/pyarrow/error.pxi

@@ -32,10 +32,6 @@ class ArrowMemoryError(MemoryError, ArrowException):
     pass
 
 
-class ArrowIOError(IOError, ArrowException):
-    pass
-
-
 class ArrowKeyError(KeyError, ArrowException):
     def __str__(self):
         # Override KeyError.__str__, as it uses the repr() of the key
@@ -62,6 +58,10 @@ class ArrowSerializationError(ArrowException):
     pass
 
 
+# Compatibility alias
+ArrowIOError = IOError
+
+
 # This function could be written directly in C++ if we didn't
 # define Arrow-specific subclasses (ArrowInvalid etc.)
 cdef int check_status(const CStatus& status) nogil except -1:
@@ -73,11 +73,30 @@ cdef int check_status(const CStatus& status) nogil except -1:
             RestorePyError(status)
             return -1
 
+        # We don't use Status::ToString() as it would redundantly include
+        # the C++ class name.
         message = frombytes(status.message())
+        detail = status.detail()
+        if detail != nullptr:
+            message += ". Detail: " + frombytes(detail.get().ToString())
+
         if status.IsInvalid():
             raise ArrowInvalid(message)
         elif status.IsIOError():
-            raise ArrowIOError(message)
+            # Note: OSError constructor is
+            #   OSError(message)
+            # or
+            #   OSError(errno, message, filename=None)
+            # or (on Windows)
+            #   OSError(errno, message, filename, winerror)
+            errno = ErrnoFromStatus(status)
+            winerror = WinErrorFromStatus(status)
+            if winerror != 0:
+                raise IOError(errno, message, None, winerror)
+            elif errno != 0:
+                raise IOError(errno, message)
+            else:
+                raise IOError(message)
         elif status.IsOutOfMemory():
             raise ArrowMemoryError(message)
         elif status.IsKeyError():

python/pyarrow/includes/common.pxd

@@ -77,6 +77,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         c_string ToString()
         c_string message()
+        shared_ptr[CStatusDetail] detail()
 
         c_bool ok()
         c_bool IsIOError()
@@ -89,6 +90,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         c_bool IsIndexError()
         c_bool IsSerializationError()
 
+    cdef cppclass CStatusDetail "arrow::StatusDetail":
+        c_string ToString()
+
+
 cdef extern from "arrow/result.h" namespace "arrow" nogil:
     cdef cppclass CResult "arrow::Result"[T]:
         c_bool ok()

python/pyarrow/includes/libarrow.pxd

@@ -1684,12 +1684,15 @@ cdef extern from 'arrow/util/compression.h' namespace 'arrow' nogil:
         int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input)
 
 
+cdef extern from 'arrow/util/io_util.h' namespace 'arrow::internal' nogil:
+    int ErrnoFromStatus(CStatus status)
+    int WinErrorFromStatus(CStatus status)
+
 cdef extern from 'arrow/util/iterator.h' namespace 'arrow' nogil:
     cdef cppclass CIterator" arrow::Iterator"[T]:
         CResult[T] Next()
         CStatus Visit[Visitor](Visitor&& visitor)
 
-
 cdef extern from 'arrow/util/thread_pool.h' namespace 'arrow' nogil:
     int GetCpuThreadPoolCapacity()
     CStatus SetCpuThreadPoolCapacity(int threads)

python/pyarrow/tests/test_fs.py

@@ -24,7 +24,8 @@ except ImportError:
 import pytest
 
 import pyarrow as pa
-from pyarrow.tests.test_io import gzip_compress, gzip_decompress
+from pyarrow.tests.test_io import (gzip_compress, gzip_decompress,
+                                   assert_file_not_found)
 from pyarrow.fs import (FileType, FileSelector, FileSystem, LocalFileSystem,
                         LocalFileSystemOptions, SubTreeFileSystem,
                         _MockFileSystem)
@@ -480,6 +481,26 @@ def test_localfs_options():
         LocalFileSystem(xxx=False)
 
 
+def test_localfs_errors(localfs):
+    # Local filesystem errors should raise the right Python exceptions
+    # (e.g. FileNotFoundError)
+    fs = localfs['fs']
+    with assert_file_not_found():
+        fs.open_input_stream('/non/existent/file')
+    with assert_file_not_found():
+        fs.open_output_stream('/non/existent/file')
+    with assert_file_not_found():
+        fs.create_dir('/non/existent/dir', recursive=False)
+    with assert_file_not_found():
+        fs.delete_dir('/non/existent/dir')
+    with assert_file_not_found():
+        fs.delete_file('/non/existent/dir')
+    with assert_file_not_found():
+        fs.move('/non/existent', '/xxx')
+    with assert_file_not_found():
+        fs.copy_file('/non/existent', '/xxx')
+
+
 @pytest.mark.s3
 def test_s3_options(minio_server):
     from pyarrow.fs import S3Options

python/pyarrow/tests/test_hdfs.py

@@ -417,6 +417,7 @@ def _get_hdfs_uri(path):
     return uri
 
 
+@pytest.mark.hdfs
 @pytest.mark.pandas
 @pytest.mark.parquet
 @pytest.mark.fastparquet

python/pyarrow/tests/test_io.py

@@ -16,6 +16,7 @@
 # under the License.
 
 import bz2
+from contextlib import contextmanager
 from io import (BytesIO, StringIO, TextIOWrapper, BufferedIOBase, IOBase)
 import gc
 import gzip
@@ -55,6 +56,16 @@ def check_large_seeks(file_factory):
         os.unlink(filename)
 
 
+@contextmanager
+def assert_file_not_found():
+    if PY2:
+        with pytest.raises(IOError):
+            yield
+    else:
+        with pytest.raises(FileNotFoundError):
+            yield
+
+
 # ----------------------------------------------------------------------
 # Python file-like objects
 
@@ -1003,6 +1014,13 @@ def test_native_file_TextIOWrapper(tmpdir):
         assert res == data
 
 
+def test_native_file_open_error():
+    with assert_file_not_found():
+        pa.OSFile('non_existent_file', 'rb')
+    with assert_file_not_found():
+        pa.memory_map('non_existent_file', 'rb')
+
+
 # ----------------------------------------------------------------------
 # Buffered streams
 
@@ -1404,7 +1422,7 @@ def test_input_stream_errors(tmpdir):
         with pytest.raises(TypeError):
             pa.input_stream(arg)
 
-    with pytest.raises(IOError):
+    with assert_file_not_found():
         pa.input_stream("non_existent_file")
 
     with open(str(tmpdir / 'new_file'), 'wb') as f:

python/pyarrow/tests/test_parquet.py

@@ -660,12 +660,9 @@ def test_compression_level():
                             ("None", 444), ("lzo", 14)]
     buf = io.BytesIO()
     for (codec, level) in invalid_combinations:
-        try:
+        with pytest.raises(IOError):
             _write_table(table, buf, compression=codec,
                          compression_level=level)
-            assert 0
-        except pa.ArrowException:
-            pass
 
 
 @pytest.mark.pandas