mirror of
https://github.com/apache/arrow.git
synced 2026-10-29 09:05:47 +08:00
ARROW-5389: [C++] Add Temporary Directory facility
This commit is contained in:
1 parent
cee3fc62a7
commit
74dcf50be4
7 files changed
+389
-111
No files matched your search
@@ -33,6 +33,7 @@ add_arrow_test(concatenate-test)
|
||||
add_arrow_test(decimal-test)
|
||||
add_arrow_test(hashing-test)
|
||||
add_arrow_test(int-util-test)
|
||||
add_arrow_test(io-util-test)
|
||||
add_arrow_test(key-value-metadata-test)
|
||||
add_arrow_test(lazy-test)
|
||||
add_arrow_test(logging-test)
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "arrow/testing/gtest_util.h"
|
||||
#include "arrow/util/io-util.h"
|
||||
|
||||
namespace arrow {
|
||||
namespace internal {
|
||||
|
||||
void AssertExists(const PlatformFilename& path) {
|
||||
bool exists = false;
|
||||
ASSERT_OK(FileExists(path, &exists));
|
||||
ASSERT_TRUE(exists) << "Path '" << path.ToString() << "' doesn't exist";
|
||||
}
|
||||
|
||||
void AssertNotExists(const PlatformFilename& path) {
|
||||
bool exists = true;
|
||||
ASSERT_OK(FileExists(path, &exists));
|
||||
ASSERT_FALSE(exists) << "Path '" << path.ToString() << "' exists";
|
||||
}
|
||||
|
||||
TEST(CreateDirDeleteDir, Basics) {
|
||||
const std::string BASE = "xxx-io-util-test-dir";
|
||||
bool created, deleted;
|
||||
PlatformFilename parent, child;
|
||||
|
||||
ASSERT_OK(PlatformFilename::FromString(BASE, &parent));
|
||||
ASSERT_EQ(parent.ToString(), BASE);
|
||||
|
||||
// Make sure the directory doesn't exist already
|
||||
ARROW_UNUSED(DeleteDirTree(parent));
|
||||
|
||||
AssertNotExists(parent);
|
||||
|
||||
ASSERT_OK(CreateDir(parent, &created));
|
||||
ASSERT_TRUE(created);
|
||||
AssertExists(parent);
|
||||
ASSERT_OK(CreateDir(parent, &created));
|
||||
ASSERT_FALSE(created); // already exists
|
||||
AssertExists(parent);
|
||||
|
||||
ASSERT_OK(PlatformFilename::FromString(BASE + "/some-child", &child));
|
||||
ASSERT_OK(CreateDir(child, &created));
|
||||
ASSERT_TRUE(created);
|
||||
AssertExists(child);
|
||||
|
||||
ASSERT_OK(DeleteDirTree(parent, &deleted));
|
||||
ASSERT_TRUE(deleted);
|
||||
AssertNotExists(parent);
|
||||
AssertNotExists(child);
|
||||
|
||||
// Parent is deleted, cannot create child again
|
||||
ASSERT_RAISES(IOError, CreateDir(child, &created));
|
||||
|
||||
// It's not an error to call DeleteDirTree on a non-existent path.
|
||||
ASSERT_OK(DeleteDirTree(parent, &deleted));
|
||||
ASSERT_FALSE(deleted);
|
||||
}
|
||||
|
||||
TEST(TemporaryDir, Basics) {
|
||||
std::unique_ptr<TemporaryDir> dir;
|
||||
PlatformFilename fn;
|
||||
|
||||
ASSERT_OK(TemporaryDir::Make("some-prefix-", &dir));
|
||||
fn = dir->path();
|
||||
// Path has a trailing separator, for convenience
|
||||
ASSERT_EQ(fn.ToString().back(), '/');
|
||||
#if defined(_MSC_VER)
|
||||
ASSERT_EQ(fn.ToNative().back(), L'/');
|
||||
#else
|
||||
ASSERT_EQ(fn.ToNative().back(), '/');
|
||||
#endif
|
||||
AssertExists(fn);
|
||||
ASSERT_NE(fn.ToString().find("some-prefix-"), std::string::npos);
|
||||
|
||||
// Create child contents to check that they're cleaned up at the end
|
||||
#if defined(_MSC_VER)
|
||||
PlatformFilename child(fn.ToNative() + L"some-child");
|
||||
#else
|
||||
PlatformFilename child(fn.ToNative() + "some-child");
|
||||
#endif
|
||||
ASSERT_OK(CreateDir(child));
|
||||
AssertExists(child);
|
||||
|
||||
dir.reset();
|
||||
AssertNotExists(fn);
|
||||
AssertNotExists(child);
|
||||
}
|
||||
|
||||
TEST(DeleteFile, Basics) {
|
||||
std::unique_ptr<TemporaryDir> dir;
|
||||
PlatformFilename fn;
|
||||
int fd;
|
||||
bool deleted;
|
||||
|
||||
ASSERT_OK(TemporaryDir::Make("io-util-test-", &dir));
|
||||
ASSERT_OK(dir->path().Join("test-file", &fn));
|
||||
|
||||
AssertNotExists(fn);
|
||||
ASSERT_OK(FileOpenWritable(fn, true /* write_only */, true /* truncate */,
|
||||
false /* append */, &fd));
|
||||
ASSERT_OK(FileClose(fd));
|
||||
AssertExists(fn);
|
||||
ASSERT_OK(DeleteFile(fn, &deleted));
|
||||
ASSERT_TRUE(deleted);
|
||||
AssertNotExists(fn);
|
||||
ASSERT_OK(DeleteFile(fn, &deleted));
|
||||
ASSERT_FALSE(deleted);
|
||||
AssertNotExists(fn);
|
||||
|
||||
// Cannot call DeleteFile on directory
|
||||
ASSERT_OK(dir->path().Join("test-dir", &fn));
|
||||
ASSERT_OK(CreateDir(fn));
|
||||
AssertExists(fn);
|
||||
ASSERT_RAISES(IOError, DeleteFile(fn));
|
||||
}
|
||||
|
||||
} // namespace internal
|
||||
} // namespace arrow
|
||||
+135
-48
@@ -53,15 +53,7 @@
|
||||
#include <stdexcept>
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#define USE_BOOST_FILESYSTEM 1
|
||||
#else
|
||||
#define USE_BOOST_FILESYSTEM 0
|
||||
#endif
|
||||
|
||||
#if USE_BOOST_FILESYSTEM
|
||||
#include <boost/filesystem.hpp> // NOLINT
|
||||
#endif
|
||||
#include <boost/filesystem.hpp>
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// file compatibility stuff
|
||||
@@ -105,6 +97,7 @@
|
||||
|
||||
#include "arrow/buffer.h"
|
||||
#include "arrow/util/io-util.h"
|
||||
#include "arrow/util/logging.h"
|
||||
|
||||
namespace arrow {
|
||||
namespace io {
|
||||
@@ -192,24 +185,40 @@ Status StdinStream::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
|
||||
|
||||
namespace internal {
|
||||
|
||||
namespace bfs = ::boost::filesystem;
|
||||
|
||||
#define BOOST_FILESYSTEM_TRY try {
|
||||
#define BOOST_FILESYSTEM_CATCH \
|
||||
} \
|
||||
catch (bfs::filesystem_error & _err) { \
|
||||
return ToStatus(_err); \
|
||||
}
|
||||
|
||||
// NOTE: catching filesystem_error gives more context than system::error_code
|
||||
// (it includes the file path(s) in the error message)
|
||||
|
||||
static Status ToStatus(const bfs::filesystem_error& err) {
|
||||
return Status::IOError(err.what());
|
||||
}
|
||||
|
||||
//
|
||||
// PlatformFilename implementation
|
||||
//
|
||||
|
||||
struct PlatformFilename::Impl {
|
||||
#if USE_BOOST_FILESYSTEM
|
||||
::boost::filesystem::path path;
|
||||
#else
|
||||
std::string path; // 8-bit Unix path
|
||||
#endif
|
||||
bfs::path path;
|
||||
};
|
||||
|
||||
PlatformFilename::PlatformFilename() : impl_(new Impl{}) {}
|
||||
|
||||
PlatformFilename::~PlatformFilename() {}
|
||||
|
||||
PlatformFilename::PlatformFilename(const Impl& impl) : impl_(new Impl(impl)) {}
|
||||
|
||||
PlatformFilename::PlatformFilename(Impl&& impl) : impl_(new Impl(std::move(impl))) {}
|
||||
|
||||
PlatformFilename::PlatformFilename(const PlatformFilename& other)
|
||||
: impl_(new Impl{other.impl_->path}) {}
|
||||
: PlatformFilename(Impl{other.impl_->path}) {}
|
||||
|
||||
PlatformFilename::PlatformFilename(PlatformFilename&& other)
|
||||
: impl_(std::move(other.impl_)) {}
|
||||
@@ -224,41 +233,107 @@ PlatformFilename& PlatformFilename::operator=(PlatformFilename&& other) {
|
||||
return *this;
|
||||
}
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
PlatformFilename::PlatformFilename(const std::wstring& path) : impl_(new Impl{path}) {}
|
||||
#else
|
||||
PlatformFilename::PlatformFilename(const std::string& path) : impl_(new Impl{path}) {}
|
||||
#endif
|
||||
PlatformFilename::PlatformFilename(const PlatformFilename::NativePathString& path)
|
||||
: PlatformFilename(Impl{path}) {}
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
const std::wstring& PlatformFilename::ToNative() const { return impl_->path.native(); }
|
||||
#elif USE_BOOST_FILESYSTEM
|
||||
const std::string& PlatformFilename::ToNative() const { return impl_->path.native(); }
|
||||
#else
|
||||
const std::string& PlatformFilename::ToNative() const { return impl_->path; }
|
||||
#endif
|
||||
const PlatformFilename::NativePathString& PlatformFilename::ToNative() const {
|
||||
return impl_->path.native();
|
||||
}
|
||||
|
||||
#if USE_BOOST_FILESYSTEM
|
||||
std::string PlatformFilename::ToString() const { return impl_->path.string(); }
|
||||
#else
|
||||
std::string PlatformFilename::ToString() const { return impl_->path; }
|
||||
#endif
|
||||
|
||||
Status PlatformFilename::FromString(const std::string& file_name, PlatformFilename* out) {
|
||||
namespace {
|
||||
|
||||
Status StringToNative(const std::string& s, PlatformFilename::NativePathString* out) {
|
||||
#if defined(_MSC_VER)
|
||||
try {
|
||||
auto wpath =
|
||||
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>>{}.from_bytes(file_name);
|
||||
*out = PlatformFilename(std::move(wpath));
|
||||
*out = std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>>{}.from_bytes(s);
|
||||
} catch (std::range_error& e) {
|
||||
return Status::Invalid(e.what());
|
||||
}
|
||||
#else
|
||||
*out = PlatformFilename(file_name);
|
||||
*out = s;
|
||||
#endif
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Status PlatformFilename::FromString(const std::string& file_name, PlatformFilename* out) {
|
||||
NativePathString ns;
|
||||
RETURN_NOT_OK(StringToNative(file_name, &ns));
|
||||
*out = PlatformFilename(std::move(ns));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status PlatformFilename::Join(const std::string& child_name,
|
||||
PlatformFilename* out) const {
|
||||
NativePathString ns;
|
||||
RETURN_NOT_OK(StringToNative(child_name, &ns));
|
||||
auto path = impl_->path / ns;
|
||||
*out = PlatformFilename(Impl{std::move(path)});
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status CreateDir(const PlatformFilename& dir_path, bool* created) {
|
||||
bool res;
|
||||
BOOST_FILESYSTEM_TRY
|
||||
res = bfs::create_directory(dir_path.impl_->path);
|
||||
BOOST_FILESYSTEM_CATCH
|
||||
if (created) {
|
||||
*created = res;
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status DeleteDirTree(const PlatformFilename& dir_path, bool* deleted) {
|
||||
BOOST_FILESYSTEM_TRY
|
||||
auto n_removed = bfs::remove_all(dir_path.impl_->path);
|
||||
if (deleted) {
|
||||
*deleted = n_removed != 0;
|
||||
}
|
||||
BOOST_FILESYSTEM_CATCH
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status DeleteFile(const PlatformFilename& file_path, bool* deleted) {
|
||||
BOOST_FILESYSTEM_TRY
|
||||
bool res = false;
|
||||
const auto& path = file_path.impl_->path;
|
||||
// XXX There is a race here, and boost::filesystem doesn't allow deleting
|
||||
// only files and not empty directories.
|
||||
auto st = bfs::symlink_status(path);
|
||||
if (!bfs::is_directory(st)) {
|
||||
res = bfs::remove(path);
|
||||
} else {
|
||||
return Status::IOError("Cannot delete directory '", path.string());
|
||||
}
|
||||
if (deleted) {
|
||||
*deleted = res;
|
||||
}
|
||||
BOOST_FILESYSTEM_CATCH
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status FileExists(const PlatformFilename& path, bool* out) {
|
||||
BOOST_FILESYSTEM_TRY
|
||||
*out = bfs::exists(path.impl_->path);
|
||||
BOOST_FILESYSTEM_CATCH
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
//
|
||||
// File name handling
|
||||
//
|
||||
|
||||
Status FileNameFromString(const std::string& file_name, PlatformFilename* out) {
|
||||
return PlatformFilename::FromString(file_name, out);
|
||||
}
|
||||
|
||||
//
|
||||
// Functions for creating file descriptors
|
||||
//
|
||||
|
||||
#define CHECK_LSEEK(retval) \
|
||||
if ((retval) == -1) return Status::IOError("lseek failed");
|
||||
|
||||
@@ -280,18 +355,6 @@ static inline Status CheckFileOpResult(int ret, int errno_actual,
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
//
|
||||
// File name handling
|
||||
//
|
||||
|
||||
Status FileNameFromString(const std::string& file_name, PlatformFilename* out) {
|
||||
return PlatformFilename::FromString(file_name, out);
|
||||
}
|
||||
|
||||
//
|
||||
// Functions for creating file descriptors
|
||||
//
|
||||
|
||||
Status FileOpenReadable(const PlatformFilename& file_name, int* fd) {
|
||||
int ret, errno_actual;
|
||||
#if defined(_MSC_VER)
|
||||
@@ -724,5 +787,29 @@ Status DelEnvVar(const char* name) {
|
||||
|
||||
Status DelEnvVar(const std::string& name) { return DelEnvVar(name.c_str()); }
|
||||
|
||||
TemporaryDir::TemporaryDir(PlatformFilename&& path) : path_(std::move(path)) {}
|
||||
|
||||
TemporaryDir::~TemporaryDir() { DCHECK_OK(DeleteDirTree(path_)); }
|
||||
|
||||
Status TemporaryDir::Make(const std::string& prefix, std::unique_ptr<TemporaryDir>* out) {
|
||||
bfs::path path;
|
||||
|
||||
BOOST_FILESYSTEM_TRY
|
||||
auto model = bfs::path(prefix + "%%%%-%%%%-%%%%-%%%%");
|
||||
path = bfs::temp_directory_path() / bfs::unique_path(model);
|
||||
path += "/";
|
||||
BOOST_FILESYSTEM_CATCH
|
||||
|
||||
PlatformFilename fn(path.native());
|
||||
bool created;
|
||||
RETURN_NOT_OK(CreateDir(fn, &created));
|
||||
if (!created) {
|
||||
// XXX Should we retry?
|
||||
return Status::IOError("Path already exists: '", fn.ToString(), "'");
|
||||
}
|
||||
out->reset(new TemporaryDir(std::move(fn)));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
} // namespace internal
|
||||
} // namespace arrow
|
||||
@@ -23,6 +23,13 @@
|
||||
|
||||
#include "arrow/io/interfaces.h"
|
||||
#include "arrow/status.h"
|
||||
#include "arrow/util/macros.h"
|
||||
|
||||
// The Windows API defines DeleteFile as a macro resolving to either
|
||||
// DeleteFileA or DeleteFileW. Need to undo it.
|
||||
#if defined(_WIN32) && defined(DeleteFile)
|
||||
#undef DeleteFile
|
||||
#endif
|
||||
|
||||
namespace arrow {
|
||||
|
||||
@@ -89,33 +96,50 @@ namespace internal {
|
||||
|
||||
class ARROW_EXPORT PlatformFilename {
|
||||
public:
|
||||
#if defined(_MSC_VER)
|
||||
using NativePathString = std::wstring;
|
||||
#else
|
||||
using NativePathString = std::string;
|
||||
#endif
|
||||
|
||||
~PlatformFilename();
|
||||
PlatformFilename();
|
||||
PlatformFilename(const PlatformFilename&);
|
||||
PlatformFilename(PlatformFilename&&);
|
||||
PlatformFilename& operator=(const PlatformFilename&);
|
||||
PlatformFilename& operator=(PlatformFilename&&);
|
||||
explicit PlatformFilename(const NativePathString& path);
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
const std::wstring& ToNative() const;
|
||||
#else
|
||||
const std::string& ToNative() const;
|
||||
#endif
|
||||
const NativePathString& ToNative() const;
|
||||
std::string ToString() const;
|
||||
|
||||
// These functions can fail for character encoding reasons.
|
||||
static Status FromString(const std::string& file_name, PlatformFilename* out);
|
||||
Status Join(const std::string& child_name, PlatformFilename* out) const;
|
||||
|
||||
private:
|
||||
struct Impl;
|
||||
std::unique_ptr<Impl> impl_;
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
explicit PlatformFilename(const std::wstring& path);
|
||||
#else
|
||||
explicit PlatformFilename(const std::string& path);
|
||||
#endif
|
||||
explicit PlatformFilename(const Impl& impl);
|
||||
explicit PlatformFilename(Impl&& impl);
|
||||
|
||||
// Those functions need access to the embedded path object
|
||||
friend ARROW_EXPORT Status CreateDir(const PlatformFilename&, bool*);
|
||||
friend ARROW_EXPORT Status DeleteDirTree(const PlatformFilename&, bool*);
|
||||
friend ARROW_EXPORT Status DeleteFile(const PlatformFilename&, bool*);
|
||||
friend ARROW_EXPORT Status FileExists(const PlatformFilename&, bool*);
|
||||
};
|
||||
|
||||
ARROW_EXPORT
|
||||
Status CreateDir(const PlatformFilename& dir_path, bool* created = NULLPTR);
|
||||
ARROW_EXPORT
|
||||
Status DeleteDirTree(const PlatformFilename& dir_path, bool* deleted = NULLPTR);
|
||||
ARROW_EXPORT
|
||||
Status DeleteFile(const PlatformFilename& file_path, bool* deleted = NULLPTR);
|
||||
ARROW_EXPORT
|
||||
Status FileExists(const PlatformFilename& path, bool* out);
|
||||
|
||||
ARROW_EXPORT
|
||||
Status FileNameFromString(const std::string& file_name, PlatformFilename* out);
|
||||
|
||||
@@ -167,6 +191,20 @@ Status DelEnvVar(const char* name);
|
||||
ARROW_EXPORT
|
||||
Status DelEnvVar(const std::string& name);
|
||||
|
||||
class ARROW_EXPORT TemporaryDir {
|
||||
public:
|
||||
~TemporaryDir();
|
||||
|
||||
const PlatformFilename& path() { return path_; }
|
||||
|
||||
static Status Make(const std::string& prefix, std::unique_ptr<TemporaryDir>* out);
|
||||
|
||||
private:
|
||||
PlatformFilename path_;
|
||||
|
||||
explicit TemporaryDir(PlatformFilename&&);
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
} // namespace arrow
|
||||
|
||||
|
||||
@@ -22,13 +22,13 @@
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <random>
|
||||
#include <memory>
|
||||
#include <thread>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "arrow/testing/gtest_util.h"
|
||||
#include "arrow/util/io-util.h"
|
||||
|
||||
#include "plasma/client.h"
|
||||
#include "plasma/common.h"
|
||||
@@ -38,6 +38,8 @@
|
||||
|
||||
namespace plasma {
|
||||
|
||||
using arrow::internal::TemporaryDir;
|
||||
|
||||
std::string test_executable; // NOLINT
|
||||
|
||||
void AssertObjectBufferEqual(const ObjectBuffer& object_buffer,
|
||||
@@ -51,11 +53,10 @@ class TestPlasmaStore : public ::testing::Test {
|
||||
public:
|
||||
// TODO(pcm): At the moment, stdout of the test gets mixed up with
|
||||
// stdout of the object store. Consider changing that.
|
||||
|
||||
void SetUp() {
|
||||
uint64_t seed = std::chrono::high_resolution_clock::now().time_since_epoch().count();
|
||||
std::mt19937 rng(static_cast<uint32_t>(seed));
|
||||
std::string store_index = std::to_string(rng());
|
||||
store_socket_name_ = "/tmp/store" + store_index;
|
||||
ARROW_CHECK_OK(TemporaryDir::Make("plasma-client-tests-", &temp_dir_));
|
||||
store_socket_name_ = temp_dir_->path().ToString() + "store";
|
||||
|
||||
std::string plasma_directory =
|
||||
test_executable.substr(0, test_executable.find_last_of("/"));
|
||||
@@ -66,6 +67,7 @@ class TestPlasmaStore : public ::testing::Test {
|
||||
ARROW_CHECK_OK(client_.Connect(store_socket_name_, ""));
|
||||
ARROW_CHECK_OK(client2_.Connect(store_socket_name_, ""));
|
||||
}
|
||||
|
||||
virtual void TearDown() {
|
||||
ARROW_CHECK_OK(client_.Disconnect());
|
||||
ARROW_CHECK_OK(client2_.Disconnect());
|
||||
@@ -98,11 +100,10 @@ class TestPlasmaStore : public ::testing::Test {
|
||||
}
|
||||
}
|
||||
|
||||
const std::string& GetStoreSocketName() const { return store_socket_name_; }
|
||||
|
||||
protected:
|
||||
PlasmaClient client_;
|
||||
PlasmaClient client2_;
|
||||
std::unique_ptr<TemporaryDir> temp_dir_;
|
||||
std::string store_socket_name_;
|
||||
};
|
||||
|
||||
|
||||
@@ -22,13 +22,13 @@
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <random>
|
||||
#include <memory>
|
||||
#include <thread>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "arrow/testing/gtest_util.h"
|
||||
#include "arrow/util/io-util.h"
|
||||
|
||||
#include "plasma/client.h"
|
||||
#include "plasma/common.h"
|
||||
@@ -39,6 +39,8 @@
|
||||
|
||||
namespace plasma {
|
||||
|
||||
using arrow::internal::TemporaryDir;
|
||||
|
||||
std::string external_test_executable; // NOLINT
|
||||
|
||||
void AssertObjectBufferEqual(const ObjectBuffer& object_buffer,
|
||||
@@ -52,10 +54,8 @@ class TestPlasmaStoreWithExternal : public ::testing::Test {
|
||||
// TODO(pcm): At the moment, stdout of the test gets mixed up with
|
||||
// stdout of the object store. Consider changing that.
|
||||
void SetUp() override {
|
||||
uint64_t seed = std::chrono::high_resolution_clock::now().time_since_epoch().count();
|
||||
std::mt19937 rng(static_cast<uint32_t>(seed));
|
||||
std::string store_index = std::to_string(rng());
|
||||
store_socket_name_ = "/tmp/store_with_external" + store_index;
|
||||
ARROW_CHECK_OK(TemporaryDir::Make("plasma-client-tests-", &temp_dir_));
|
||||
store_socket_name_ = temp_dir_->path().ToString() + "store_with_external";
|
||||
|
||||
std::string plasma_directory =
|
||||
external_test_executable.substr(0, external_test_executable.find_last_of('/'));
|
||||
@@ -67,6 +67,7 @@ class TestPlasmaStoreWithExternal : public ::testing::Test {
|
||||
PLASMA_CHECK_SYSTEM(system(plasma_command.c_str()));
|
||||
ARROW_CHECK_OK(client_.Connect(store_socket_name_, ""));
|
||||
}
|
||||
|
||||
void TearDown() override {
|
||||
ARROW_CHECK_OK(client_.Disconnect());
|
||||
// Kill plasma_store process that we started
|
||||
@@ -85,6 +86,7 @@ class TestPlasmaStoreWithExternal : public ::testing::Test {
|
||||
|
||||
protected:
|
||||
PlasmaClient client_;
|
||||
std::unique_ptr<TemporaryDir> temp_dir_;
|
||||
std::string store_socket_name_;
|
||||
};
|
||||
|
||||
|
||||
@@ -15,12 +15,15 @@
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "arrow/testing/gtest_util.h"
|
||||
#include "arrow/util/io-util.h"
|
||||
|
||||
#include "plasma/common.h"
|
||||
#include "plasma/io.h"
|
||||
@@ -32,17 +35,7 @@ namespace fb = plasma::flatbuf;
|
||||
|
||||
namespace plasma {
|
||||
|
||||
/**
|
||||
* Create a temporary file. Needs to be closed by the caller.
|
||||
*
|
||||
* @return File descriptor of the file.
|
||||
*/
|
||||
int create_temp_file(void) {
|
||||
static char temp[] = "/tmp/tempfileXXXXXX";
|
||||
char file_name[32];
|
||||
strncpy(file_name, temp, 32);
|
||||
return mkstemp(file_name);
|
||||
}
|
||||
using arrow::internal::TemporaryDir;
|
||||
|
||||
/**
|
||||
* Seek to the beginning of a file and read a message from it.
|
||||
@@ -77,8 +70,29 @@ PlasmaObject random_plasma_object(void) {
|
||||
return object;
|
||||
}
|
||||
|
||||
TEST(PlasmaSerialization, CreateRequest) {
|
||||
int fd = create_temp_file();
|
||||
class TestPlasmaSerialization : public ::testing::Test {
|
||||
public:
|
||||
void SetUp() { ARROW_CHECK_OK(TemporaryDir::Make("plasma-ser-tests-", &temp_dir_)); }
|
||||
|
||||
// Create a temporary file.
|
||||
// A fd is returned which must be closed manually. The file itself
|
||||
// is deleted at the end of the test.
|
||||
int CreateTemporaryFile(void) {
|
||||
char path[1024];
|
||||
|
||||
std::stringstream ss;
|
||||
ss << temp_dir_->path().ToString() << "fileXXXXXX";
|
||||
strncpy(path, ss.str().c_str(), sizeof(path));
|
||||
ARROW_LOG(INFO) << "file path: '" << path << "'";
|
||||
return mkstemp(path);
|
||||
}
|
||||
|
||||
protected:
|
||||
std::unique_ptr<TemporaryDir> temp_dir_;
|
||||
};
|
||||
|
||||
TEST_F(TestPlasmaSerialization, CreateRequest) {
|
||||
int fd = CreateTemporaryFile();
|
||||
ObjectID object_id1 = random_object_id();
|
||||
int64_t data_size1 = 42;
|
||||
int64_t metadata_size1 = 11;
|
||||
@@ -99,8 +113,8 @@ TEST(PlasmaSerialization, CreateRequest) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
TEST(PlasmaSerialization, CreateReply) {
|
||||
int fd = create_temp_file();
|
||||
TEST_F(TestPlasmaSerialization, CreateReply) {
|
||||
int fd = CreateTemporaryFile();
|
||||
ObjectID object_id1 = random_object_id();
|
||||
PlasmaObject object1 = random_plasma_object();
|
||||
int64_t mmap_size1 = 1000000;
|
||||
@@ -119,8 +133,8 @@ TEST(PlasmaSerialization, CreateReply) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
TEST(PlasmaSerialization, SealRequest) {
|
||||
int fd = create_temp_file();
|
||||
TEST_F(TestPlasmaSerialization, SealRequest) {
|
||||
int fd = CreateTemporaryFile();
|
||||
ObjectID object_id1 = random_object_id();
|
||||
unsigned char digest1[kDigestSize];
|
||||
memset(&digest1[0], 7, kDigestSize);
|
||||
@@ -134,8 +148,8 @@ TEST(PlasmaSerialization, SealRequest) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
TEST(PlasmaSerialization, SealReply) {
|
||||
int fd = create_temp_file();
|
||||
TEST_F(TestPlasmaSerialization, SealReply) {
|
||||
int fd = CreateTemporaryFile();
|
||||
ObjectID object_id1 = random_object_id();
|
||||
ASSERT_OK(SendSealReply(fd, object_id1, PlasmaError::ObjectExists));
|
||||
std::vector<uint8_t> data = read_message_from_file(fd, MessageType::PlasmaSealReply);
|
||||
@@ -146,8 +160,8 @@ TEST(PlasmaSerialization, SealReply) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
TEST(PlasmaSerialization, GetRequest) {
|
||||
int fd = create_temp_file();
|
||||
TEST_F(TestPlasmaSerialization, GetRequest) {
|
||||
int fd = CreateTemporaryFile();
|
||||
ObjectID object_ids[2];
|
||||
object_ids[0] = random_object_id();
|
||||
object_ids[1] = random_object_id();
|
||||
@@ -164,8 +178,8 @@ TEST(PlasmaSerialization, GetRequest) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
TEST(PlasmaSerialization, GetReply) {
|
||||
int fd = create_temp_file();
|
||||
TEST_F(TestPlasmaSerialization, GetReply) {
|
||||
int fd = CreateTemporaryFile();
|
||||
ObjectID object_ids[2];
|
||||
object_ids[0] = random_object_id();
|
||||
object_ids[1] = random_object_id();
|
||||
@@ -199,8 +213,8 @@ TEST(PlasmaSerialization, GetReply) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
TEST(PlasmaSerialization, ReleaseRequest) {
|
||||
int fd = create_temp_file();
|
||||
TEST_F(TestPlasmaSerialization, ReleaseRequest) {
|
||||
int fd = CreateTemporaryFile();
|
||||
ObjectID object_id1 = random_object_id();
|
||||
ASSERT_OK(SendReleaseRequest(fd, object_id1));
|
||||
std::vector<uint8_t> data =
|
||||
@@ -211,8 +225,8 @@ TEST(PlasmaSerialization, ReleaseRequest) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
TEST(PlasmaSerialization, ReleaseReply) {
|
||||
int fd = create_temp_file();
|
||||
TEST_F(TestPlasmaSerialization, ReleaseReply) {
|
||||
int fd = CreateTemporaryFile();
|
||||
ObjectID object_id1 = random_object_id();
|
||||
ASSERT_OK(SendReleaseReply(fd, object_id1, PlasmaError::ObjectExists));
|
||||
std::vector<uint8_t> data = read_message_from_file(fd, MessageType::PlasmaReleaseReply);
|
||||
@@ -223,8 +237,8 @@ TEST(PlasmaSerialization, ReleaseReply) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
TEST(PlasmaSerialization, DeleteRequest) {
|
||||
int fd = create_temp_file();
|
||||
TEST_F(TestPlasmaSerialization, DeleteRequest) {
|
||||
int fd = CreateTemporaryFile();
|
||||
ObjectID object_id1 = random_object_id();
|
||||
ASSERT_OK(SendDeleteRequest(fd, std::vector<ObjectID>{object_id1}));
|
||||
std::vector<uint8_t> data =
|
||||
@@ -236,8 +250,8 @@ TEST(PlasmaSerialization, DeleteRequest) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
TEST(PlasmaSerialization, DeleteReply) {
|
||||
int fd = create_temp_file();
|
||||
TEST_F(TestPlasmaSerialization, DeleteReply) {
|
||||
int fd = CreateTemporaryFile();
|
||||
ObjectID object_id1 = random_object_id();
|
||||
PlasmaError error1 = PlasmaError::ObjectExists;
|
||||
ASSERT_OK(SendDeleteReply(fd, std::vector<ObjectID>{object_id1},
|
||||
@@ -254,8 +268,8 @@ TEST(PlasmaSerialization, DeleteReply) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
TEST(PlasmaSerialization, EvictRequest) {
|
||||
int fd = create_temp_file();
|
||||
TEST_F(TestPlasmaSerialization, EvictRequest) {
|
||||
int fd = CreateTemporaryFile();
|
||||
int64_t num_bytes = 111;
|
||||
ASSERT_OK(SendEvictRequest(fd, num_bytes));
|
||||
std::vector<uint8_t> data = read_message_from_file(fd, MessageType::PlasmaEvictRequest);
|
||||
@@ -265,8 +279,8 @@ TEST(PlasmaSerialization, EvictRequest) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
TEST(PlasmaSerialization, EvictReply) {
|
||||
int fd = create_temp_file();
|
||||
TEST_F(TestPlasmaSerialization, EvictReply) {
|
||||
int fd = CreateTemporaryFile();
|
||||
int64_t num_bytes = 111;
|
||||
ASSERT_OK(SendEvictReply(fd, num_bytes));
|
||||
std::vector<uint8_t> data = read_message_from_file(fd, MessageType::PlasmaEvictReply);
|
||||
@@ -276,8 +290,8 @@ TEST(PlasmaSerialization, EvictReply) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
TEST(PlasmaSerialization, DataRequest) {
|
||||
int fd = create_temp_file();
|
||||
TEST_F(TestPlasmaSerialization, DataRequest) {
|
||||
int fd = CreateTemporaryFile();
|
||||
ObjectID object_id1 = random_object_id();
|
||||
const char* address1 = "address1";
|
||||
int port1 = 12345;
|
||||
@@ -295,8 +309,8 @@ TEST(PlasmaSerialization, DataRequest) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
TEST(PlasmaSerialization, DataReply) {
|
||||
int fd = create_temp_file();
|
||||
TEST_F(TestPlasmaSerialization, DataReply) {
|
||||
int fd = CreateTemporaryFile();
|
||||
ObjectID object_id1 = random_object_id();
|
||||
int64_t object_size1 = 146;
|
||||
int64_t metadata_size1 = 198;
|
||||
|
||||
Reference in new issue
Block a user