mirrors/arrow
0
0 Fork 0
mirror of https://github.com/apache/arrow.git synced 2026-10-29 09:05:47 +08:00
Code Activity

Merge unit test objects to get down to 78 unit tests total

Browse Source
This commit is contained in:
Wes McKinney committed 2019-05-31 14:56:09 -05:00
1 parent 0054ef3bbd
commit 19beb21b7b
22 files changed
+346 -303
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cpp/src/arrow/io/CMakeLists.txt
+9 -6
View File
@@ -18,17 +18,20 @@
# ----------------------------------------------------------------------
# arrow_io : Arrow IO interfaces
add_arrow_test(buffered-test PREFIX "arrow-io")
add_arrow_test(compressed-test PREFIX "arrow-io")
add_arrow_test(file-test PREFIX "arrow-io")
add_arrow_test(test
SOURCES
buffered-test.cc
compressed-test.cc
file-test.cc
memory-test.cc
readahead-test.cc
PREFIX
"arrow-io")
if(ARROW_HDFS AND NOT ARROW_BOOST_HEADER_ONLY)
add_arrow_test(hdfs-test NO_VALGRIND PREFIX "arrow-io")
endif()
add_arrow_test(memory-test PREFIX "arrow-io")
add_arrow_test(readahead-test PREFIX "arrow-io")
add_arrow_benchmark(file-benchmark PREFIX "arrow-io")
add_arrow_benchmark(memory-benchmark PREFIX "arrow-io")
cpp/src/arrow/json/CMakeLists.txt
+8 -10
View File
@@ -15,16 +15,14 @@
# specific language governing permissions and limitations
# under the License.
add_arrow_test(parser-test PREFIX "arrow-json")
add_arrow_test(chunker-test PREFIX "arrow-json")
add_arrow_test(test
SOURCES
chunker-test.cc
converter-test.cc
parser-test.cc
reader-test.cc
PREFIX
"arrow-json")
add_arrow_benchmark(parser-benchmark PREFIX "arrow-json")
add_arrow_test(converter-test PREFIX "arrow-json")
add_arrow_test(chunked-builder-test PREFIX "arrow-json")
add_arrow_test(reader-test PREFIX "arrow-json")
arrow_install_all_headers("arrow/json")
cpp/src/arrow/json/test-common.h
+1 -1
View File
@@ -168,7 +168,7 @@ inline static Status ParseFromString(ParseOptions options, string_view src_str,
return parser->Finish(parsed);
}
std::string PrettyPrint(string_view one_line) {
static inline std::string PrettyPrint(string_view one_line) {
rj::Document document;
document.Parse(one_line.data());
rj::StringBuffer sb;
cpp/src/arrow/util/CMakeLists.txt
+13 -10
View File
@@ -26,27 +26,30 @@ arrow_install_all_headers("arrow/util")
# arrow_test_main
#
add_arrow_test(utility-test
SOURCES
checked-cast-test.cc
key-value-metadata-test.cc
hashing-test.cc
int-util-test.cc
io-util-test.cc
lazy-test.cc
parsing-util-test.cc
stl-util-test.cc
trie-test.cc
utf8-util-test.cc)
add_arrow_test(bit-util-test)
add_arrow_test(checked-cast-test)
add_arrow_test(compression-test)
add_arrow_test(concatenate-test)
add_arrow_test(decimal-test)
add_arrow_test(hashing-test)
add_arrow_test(int-util-test)
add_arrow_test(io-util-test)
add_arrow_test(key-value-metadata-test)
add_arrow_test(lazy-test)
add_arrow_test(logging-test)
add_arrow_test(parsing-util-test)
add_arrow_test(rle-encoding-test)
add_arrow_test(stl-util-test)
add_arrow_test(task-group-test)
add_arrow_test(thread-pool-test)
add_arrow_test(trie-test)
if(ARROW_WITH_URIPARSER)
add_arrow_test(uri-test)
endif()
add_arrow_test(utf8-util-test)
add_arrow_benchmark(bit-util-benchmark)
add_arrow_benchmark(compression-benchmark)
cpp/src/gandiva/CMakeLists.txt
+31 -29
View File
@@ -163,42 +163,44 @@ function(ADD_GANDIVA_TEST REL_TEST_NAME)
# and uses less disk space, but in some cases we need to force static
# linking (see rationale below).
if(ARG_USE_STATIC_LINKING)
add_test_case(${REL_TEST_NAME} ${TEST_ARGUMENTS} STATIC_LINK_LIBS
${GANDIVA_STATIC_TEST_LINK_LIBS})
add_test_case(${REL_TEST_NAME}
${TEST_ARGUMENTS}
STATIC_LINK_LIBS
${GANDIVA_STATIC_TEST_LINK_LIBS}
${ARG_UNPARSED_ARGUMENTS})
else()
add_test_case(${REL_TEST_NAME} ${TEST_ARGUMENTS} STATIC_LINK_LIBS
${GANDIVA_SHARED_TEST_LINK_LIBS})
add_test_case(${REL_TEST_NAME}
${TEST_ARGUMENTS}
STATIC_LINK_LIBS
${GANDIVA_SHARED_TEST_LINK_LIBS}
${ARG_UNPARSED_ARGUMENTS})
endif()
set(TARGET_NAME gandiva-${REL_TEST_NAME})
if((TARGET ${TARGET_NAME})
AND (${REL_TEST_NAME}
MATCHES
"llvm"
OR ${REL_TEST_NAME} MATCHES "expression_registry"))
# If the unit test has llvm in its name, include llvm.
add_dependencies(${TARGET_NAME} LLVM::LLVM_INTERFACE)
target_link_libraries(${TARGET_NAME} PRIVATE LLVM::LLVM_INTERFACE)
endif()
# If the unit test has llvm in its name, include llvm.
add_dependencies(${TARGET_NAME} LLVM::LLVM_INTERFACE)
target_link_libraries(${TARGET_NAME} PRIVATE LLVM::LLVM_INTERFACE)
endfunction()
add_gandiva_test(bitmap_accumulator_test)
add_gandiva_test(engine_llvm_test)
add_gandiva_test(function_signature_test)
add_gandiva_test(function_registry_test)
add_gandiva_test(llvm_types_test)
add_gandiva_test(llvm_generator_test)
add_gandiva_test(annotator_test)
add_gandiva_test(tree_expr_test)
add_gandiva_test(expr_decomposer_test)
add_gandiva_test(expression_registry_test)
add_gandiva_test(selection_vector_test)
add_gandiva_test(lru_cache_test)
add_gandiva_test(to_date_holder_test)
add_gandiva_test(simple_arena_test)
add_gandiva_test(like_holder_test)
add_gandiva_test(decimal_type_util_test)
add_gandiva_test(internals-test
SOURCES
bitmap_accumulator_test.cc
engine_llvm_test.cc
function_signature_test.cc
function_registry_test.cc
llvm_types_test.cc
llvm_generator_test.cc
annotator_test.cc
tree_expr_test.cc
expr_decomposer_test.cc
expression_registry_test.cc
selection_vector_test.cc
lru_cache_test.cc
to_date_holder_test.cc
simple_arena_test.cc
like_holder_test.cc
decimal_type_util_test.cc)
if(ARROW_GANDIVA_JAVA)
add_subdirectory(jni)
cpp/src/gandiva/bitmap_accumulator_test.cc
-5
View File
@@ -75,9 +75,4 @@ TEST_F(TestBitMapAccumulator, TestIntersectBitMaps) {
}
}
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
} // namespace gandiva
cpp/src/gandiva/engine_llvm_test.cc
-5
View File
@@ -130,9 +130,4 @@ TEST_F(TestEngine, TestAddOptimised) {
EXPECT_EQ(add_func(my_array, 5), 17);
}
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
} // namespace gandiva
cpp/src/gandiva/expr_decomposer_test.cc
-5
View File
@@ -243,9 +243,4 @@ TEST_F(TestExprDecomposer, TestIfInCondition) {
EXPECT_EQ(decomposer.if_entries_stack_.empty(), true);
}
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
} // namespace gandiva
cpp/src/gandiva/function_registry_test.cc
-5
View File
@@ -45,9 +45,4 @@ TEST_F(TestFunctionRegistry, TestNotFound) {
EXPECT_EQ(registry_.LookupSignature(add_i32_i32_ret64), nullptr);
}
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
} // namespace gandiva
cpp/src/gandiva/function_signature_test.cc
-5
View File
@@ -97,9 +97,4 @@ TEST_F(TestFunctionSignature, TestHash) {
EXPECT_EQ(f1.Hash(), f2.Hash());
}
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
} // namespace gandiva
cpp/src/gandiva/like_holder_test.cc
-5
View File
@@ -125,9 +125,4 @@ TEST_F(TestLikeHolder, TestOptimise) {
EXPECT_EQ(fnode.descriptor()->name(), "like");
}
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
} // namespace gandiva
cpp/src/gandiva/llvm_generator_test.cc
-5
View File
@@ -116,9 +116,4 @@ TEST_F(TestLLVMGenerator, TestAdd) {
}
}
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
} // namespace gandiva
cpp/src/gandiva/llvm_types_test.cc
-5
View File
@@ -57,9 +57,4 @@ TEST_F(TestLLVMTypes, TestNotFound) {
EXPECT_EQ(types_->DataVecType(arrow::null()), nullptr);
}
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
} // namespace gandiva
cpp/src/gandiva/precompiled/CMakeLists.txt
+33 -35
View File
@@ -100,43 +100,41 @@ add_custom_command(OUTPUT ${GANDIVA_PRECOMPILED_CC_PATH}
add_custom_target(precompiled ALL
DEPENDS ${GANDIVA_PRECOMPILED_BC_PATH} ${GANDIVA_PRECOMPILED_CC_PATH})
function(add_precompiled_unit_test REL_TEST_NAME)
get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
set(TEST_NAME "gandiva-precompiled-${TEST_NAME}")
add_executable(${TEST_NAME} ${REL_TEST_NAME} ${ARGN})
target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_SOURCE_DIR}/src)
target_link_libraries(${TEST_NAME} PRIVATE ${ARROW_TEST_LINK_LIBS} RE2::re2)
target_compile_definitions(${TEST_NAME}
# testing
if(ARROW_BUILD_TESTS)
add_executable(gandiva-precompiled-test
../context_helper.cc
bitmap_test.cc
bitmap.cc
epoch_time_point_test.cc
time_test.cc
time.cc
timestamp_arithmetic.cc
../cast_time.cc
../../arrow/vendored/datetime/tz.cpp
hash_test.cc
hash.cc
string_ops_test.cc
string_ops.cc
arithmetic_ops_test.cc
arithmetic_ops.cc
extended_math_ops_test.cc
extended_math_ops.cc
decimal_ops_test.cc
decimal_ops.cc
../decimal_type_util.cc
../decimal_xlarge.cc)
target_include_directories(gandiva-precompiled-test PRIVATE ${CMAKE_SOURCE_DIR}/src)
target_link_libraries(gandiva-precompiled-test PRIVATE ${ARROW_TEST_LINK_LIBS} RE2::re2)
target_compile_definitions(gandiva-precompiled-test
PRIVATE
GANDIVA_UNIT_TEST=1
ARROW_STATIC
GANDIVA_STATIC)
set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
add_test(${TEST_NAME} ${TEST_PATH})
set_property(TEST ${TEST_NAME} APPEND PROPERTY LABELS "unittest;gandiva-tests")
add_dependencies(gandiva-tests ${TEST_NAME})
endfunction(add_precompiled_unit_test REL_TEST_NAME)
# testing
if(ARROW_BUILD_TESTS)
add_precompiled_unit_test(bitmap_test.cc bitmap.cc)
add_precompiled_unit_test(epoch_time_point_test.cc)
add_precompiled_unit_test(time_test.cc
time.cc
timestamp_arithmetic.cc
../context_helper.cc
../cast_time.cc
../../arrow/vendored/datetime/tz.cpp)
add_precompiled_unit_test(hash_test.cc hash.cc)
add_precompiled_unit_test(string_ops_test.cc string_ops.cc ../context_helper.cc)
add_precompiled_unit_test(arithmetic_ops_test.cc arithmetic_ops.cc ../context_helper.cc)
add_precompiled_unit_test(extended_math_ops_test.cc extended_math_ops.cc
../context_helper.cc)
add_precompiled_unit_test(decimal_ops_test.cc
decimal_ops.cc
../decimal_type_util.cc
../decimal_xlarge.cc
../context_helper.cc)
set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/gandiva-precompiled-test")
add_test(gandiva-precompiled-test ${TEST_PATH})
set_property(TEST gandiva-precompiled-test
APPEND
PROPERTY LABELS "unittest;gandiva-tests")
add_dependencies(gandiva-tests gandiva-precompiled-test)
endif()
cpp/src/gandiva/precompiled/testing.h
+1 -1
View File
@@ -28,7 +28,7 @@
namespace gandiva {
timestamp StringToTimestamp(const char* buf) {
static inline timestamp StringToTimestamp(const char* buf) {
int64_t out = 0;
bool success = internal::ParseTimestamp(buf, "%Y-%m-%d %H:%M:%S", false, &out);
DCHECK(success);
cpp/src/gandiva/tests/test_util.h
+10 -8
View File
@@ -34,31 +34,33 @@ namespace gandiva {
// arrow/testing/gtest_util.h has good utility classes for this purpose.
// Using those
template <typename TYPE, typename C_TYPE>
static ArrayPtr MakeArrowArray(std::vector<C_TYPE> values, std::vector<bool> validity) {
static inline ArrayPtr MakeArrowArray(std::vector<C_TYPE> values,
std::vector<bool> validity) {
ArrayPtr out;
arrow::ArrayFromVector<TYPE, C_TYPE>(validity, values, &out);
return out;
}
template <typename TYPE, typename C_TYPE>
static ArrayPtr MakeArrowArray(std::vector<C_TYPE> values) {
static inline ArrayPtr MakeArrowArray(std::vector<C_TYPE> values) {
ArrayPtr out;
arrow::ArrayFromVector<TYPE, C_TYPE>(values, &out);
return out;
}
template <typename TYPE, typename C_TYPE>
static ArrayPtr MakeArrowArray(const std::shared_ptr<arrow::DataType>& type,
std::vector<C_TYPE> values, std::vector<bool> validity) {
static inline ArrayPtr MakeArrowArray(const std::shared_ptr<arrow::DataType>& type,
std::vector<C_TYPE> values,
std::vector<bool> validity) {
ArrayPtr out;
arrow::ArrayFromVector<TYPE, C_TYPE>(type, validity, values, &out);
return out;
}
template <typename TYPE, typename C_TYPE>
static ArrayPtr MakeArrowTypeArray(const std::shared_ptr<arrow::DataType>& type,
const std::vector<C_TYPE>& values,
const std::vector<bool>& validity) {
static inline ArrayPtr MakeArrowTypeArray(const std::shared_ptr<arrow::DataType>& type,
const std::vector<C_TYPE>& values,
const std::vector<bool>& validity) {
ArrayPtr out;
arrow::ArrayFromVector<TYPE, C_TYPE>(type, validity, values, &out);
return out;
@@ -92,7 +94,7 @@ static ArrayPtr MakeArrowTypeArray(const std::shared_ptr<arrow::DataType>& type,
EXPECT_TRUE((a)->Equals(b)) << "expected type: " << (a)->ToString() \
<< " actual type: " << (b)->ToString()
std::shared_ptr<Configuration> TestConfiguration() {
static inline std::shared_ptr<Configuration> TestConfiguration() {
auto builder = ConfigurationBuilder();
return builder.DefaultConfiguration();
}
cpp/src/gandiva/to_date_holder_test.cc
-5
View File
@@ -134,9 +134,4 @@ TEST_F(TestToDateHolder, TestSimpleDateTimeMakeError) {
EXPECT_EQ(status.IsInvalid(), true) << status.message();
}
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
} // namespace gandiva
cpp/src/gandiva/tree_expr_test.cc
-5
View File
@@ -156,9 +156,4 @@ TEST_F(TestExprTree, TestExpression) {
EXPECT_EQ(null_if_null->native_function(), fn);
}
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
} // namespace gandiva
cpp/src/parquet/CMakeLists.txt
+46 -21
View File
@@ -37,9 +37,6 @@ function(ADD_PARQUET_TEST REL_TEST_NAME)
"${one_value_args}"
"${multi_value_args}"
${ARGN})
if(ARG_UNPARSED_ARGUMENTS)
message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
endif()
set(TEST_ARGUMENTS PREFIX "parquet" LABELS "parquet-tests")
@@ -47,11 +44,17 @@ function(ADD_PARQUET_TEST REL_TEST_NAME)
# and uses less disk space, but in some cases we need to force static
# linking (see rationale below).
if(ARG_USE_STATIC_LINKING OR ARROW_TEST_LINKAGE STREQUAL "static")
add_test_case(${REL_TEST_NAME} STATIC_LINK_LIBS ${PARQUET_STATIC_TEST_LINK_LIBS}
${TEST_ARGUMENTS})
add_test_case(${REL_TEST_NAME}
STATIC_LINK_LIBS
${PARQUET_STATIC_TEST_LINK_LIBS}
${TEST_ARGUMENTS}
${ARG_UNPARSED_ARGUMENTS})
else()
add_test_case(${REL_TEST_NAME} STATIC_LINK_LIBS ${PARQUET_SHARED_TEST_LINK_LIBS}
${TEST_ARGUMENTS})
add_test_case(${REL_TEST_NAME}
STATIC_LINK_LIBS
${PARQUET_SHARED_TEST_LINK_LIBS}
${TEST_ARGUMENTS}
${ARG_UNPARSED_ARGUMENTS})
endif()
endfunction()
@@ -282,27 +285,49 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/parquet_version.h"
# pkg-config support
arrow_add_pkg_config("parquet")
add_parquet_test(bloom_filter-test)
add_parquet_test(column_reader-test)
add_parquet_test(column_scanner-test)
add_parquet_test(column_writer-test)
add_parquet_test(deprecated_io-test)
add_parquet_test(file-serialize-test)
add_parquet_test(properties-test)
add_parquet_test(statistics-test)
add_parquet_test(encoding-test)
add_parquet_test(metadata-test)
add_parquet_test(public-api-test)
add_parquet_test(types-test)
add_parquet_test(reader-test)
add_parquet_test(internals-test
SOURCES
bloom_filter-test.cc
deprecated_io-test.cc
properties-test.cc
statistics-test.cc
encoding-test.cc
metadata-test.cc
public-api-test.cc
types-test.cc
test-util.cc)
add_parquet_test(reader-test
SOURCES
column_reader-test.cc
column_scanner-test.cc
reader-test.cc
test-util.cc)
add_parquet_test(writer-test
SOURCES
column_writer-test.cc
file-serialize-test.cc
test-util.cc)
add_parquet_test(arrow-test
SOURCES
arrow/arrow-reader-writer-test.cc
arrow/arrow-schema-test.cc
test-util.cc)
# Those tests need to use static linking as they access thrift-generated
# symbols which are not exported by parquet.dll on Windows (PARQUET-1420).
add_parquet_test(file-deserialize-test USE_STATIC_LINKING)
add_parquet_test(file-deserialize-test
USE_STATIC_LINKING
SOURCES
file-deserialize-test.cc
test-util.cc)
add_parquet_test(schema-test USE_STATIC_LINKING)
add_parquet_benchmark(column-io-benchmark)
add_parquet_benchmark(encoding-benchmark)
add_parquet_benchmark(arrow/reader-writer-benchmark PREFIX "parquet-arrow")
# Required for tests, the ExternalProject for zstd does not build on CMake < 3.7
if(ARROW_WITH_ZSTD)
cpp/src/parquet/arrow/CMakeLists.txt
-5
View File
@@ -15,9 +15,4 @@
# specific language governing permissions and limitations
# under the License.
add_parquet_test(arrow-schema-test)
add_parquet_test(arrow-reader-writer-test)
add_parquet_benchmark(reader-writer-benchmark PREFIX "parquet-arrow")
arrow_install_all_headers("parquet/arrow")
cpp/src/parquet/test-util.cc
+136
View File
@@ -0,0 +1,136 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
// This module defines an abstract interface for iterating through pages in a
// Parquet column chunk within a row group. It could be extended in the future
// to iterate through all data pages in all chunks in a file.
#include "parquet/test-util.h"
#include <algorithm>
#include <chrono>
#include <limits>
#include <memory>
#include <random>
#include <string>
#include <utility>
#include <vector>
#include "parquet/column_page.h"
#include "parquet/column_reader.h"
#include "parquet/column_writer.h"
#include "parquet/encoding.h"
#include "parquet/platform.h"
namespace parquet {
namespace test {
const char* get_data_dir() {
const auto result = std::getenv("PARQUET_TEST_DATA");
if (!result || !result[0]) {
throw ParquetTestException(
"Please point the PARQUET_TEST_DATA environment "
"variable to the test data directory");
}
return result;
}
std::string get_bad_data_dir() {
// PARQUET_TEST_DATA should point to ARROW_HOME/cpp/submodules/parquet-testing/data
// so need to reach one folder up to access the "bad_data" folder.
std::string data_dir(get_data_dir());
std::stringstream ss;
ss << data_dir << "/../bad_data";
return ss.str();
}
std::string get_data_file(const std::string& filename, bool is_good) {
std::stringstream ss;
if (is_good) {
ss << get_data_dir();
} else {
ss << get_bad_data_dir();
}
ss << "/" << filename;
return ss.str();
}
void random_bytes(int n, uint32_t seed, std::vector<uint8_t>* out) {
std::default_random_engine gen(seed);
std::uniform_int_distribution<int> d(0, 255);
out->resize(n);
for (int i = 0; i < n; ++i) {
(*out)[i] = static_cast<uint8_t>(d(gen));
}
}
void random_bools(int n, double p, uint32_t seed, bool* out) {
std::default_random_engine gen(seed);
std::bernoulli_distribution d(p);
for (int i = 0; i < n; ++i) {
out[i] = d(gen);
}
}
void random_Int96_numbers(int n, uint32_t seed, int32_t min_value, int32_t max_value,
Int96* out) {
std::default_random_engine gen(seed);
std::uniform_int_distribution<int32_t> d(min_value, max_value);
for (int i = 0; i < n; ++i) {
out[i].value[0] = d(gen);
out[i].value[1] = d(gen);
out[i].value[2] = d(gen);
}
}
void random_fixed_byte_array(int n, uint32_t seed, uint8_t* buf, int len, FLBA* out) {
std::default_random_engine gen(seed);
std::uniform_int_distribution<int> d(0, 255);
for (int i = 0; i < n; ++i) {
out[i].ptr = buf;
for (int j = 0; j < len; ++j) {
buf[j] = static_cast<uint8_t>(d(gen));
}
buf += len;
}
}
void random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, int min_size,
int max_size) {
std::default_random_engine gen(seed);
std::uniform_int_distribution<int> d1(min_size, max_size);
std::uniform_int_distribution<int> d2(0, 255);
for (int i = 0; i < n; ++i) {
int len = d1(gen);
out[i].len = len;
out[i].ptr = buf;
for (int j = 0; j < len; ++j) {
buf[j] = static_cast<uint8_t>(d2(gen));
}
buf += len;
}
}
void random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, int max_size) {
random_byte_array(n, seed, buf, out, 0, max_size);
}
} // namespace test
} // namespace parquet
cpp/src/parquet/test-util.h
+58 -127
View File
@@ -44,7 +44,7 @@ namespace parquet {
static constexpr int FLBA_LENGTH = 12;
bool operator==(const FixedLenByteArray& a, const FixedLenByteArray& b) {
inline bool operator==(const FixedLenByteArray& a, const FixedLenByteArray& b) {
return 0 == memcmp(a.ptr, b.ptr, FLBA_LENGTH);
}
@@ -58,37 +58,10 @@ class ParquetTestException : public parquet::ParquetException {
using ParquetException::ParquetException;
};
const char* get_data_dir() {
const auto result = std::getenv("PARQUET_TEST_DATA");
if (!result || !result[0]) {
throw ParquetTestException(
"Please point the PARQUET_TEST_DATA environment "
"variable to the test data directory");
}
return result;
}
const char* get_data_dir();
std::string get_bad_data_dir();
std::string get_bad_data_dir() {
// PARQUET_TEST_DATA should point to ARROW_HOME/cpp/submodules/parquet-testing/data
// so need to reach one folder up to access the "bad_data" folder.
std::string data_dir(get_data_dir());
std::stringstream ss;
ss << data_dir << "/../bad_data";
return ss.str();
}
std::string get_data_file(const std::string& filename, bool is_good = true) {
std::stringstream ss;
if (is_good) {
ss << get_data_dir();
} else {
ss << get_bad_data_dir();
}
ss << "/" << filename;
return ss.str();
}
std::string get_data_file(const std::string& filename, bool is_good = true);
template <typename T>
static inline void assert_vector_equal(const std::vector<T>& left,
@@ -130,26 +103,11 @@ static std::vector<T> slice(const std::vector<T>& values, int start, int end) {
return out;
}
void random_bytes(int n, uint32_t seed, std::vector<uint8_t>* out) {
std::default_random_engine gen(seed);
std::uniform_int_distribution<int> d(0, 255);
out->resize(n);
for (int i = 0; i < n; ++i) {
(*out)[i] = static_cast<uint8_t>(d(gen));
}
}
void random_bools(int n, double p, uint32_t seed, bool* out) {
std::default_random_engine gen(seed);
std::bernoulli_distribution d(p);
for (int i = 0; i < n; ++i) {
out[i] = d(gen);
}
}
void random_bytes(int n, uint32_t seed, std::vector<uint8_t>* out);
void random_bools(int n, double p, uint32_t seed, bool* out);
template <typename T>
void random_numbers(int n, uint32_t seed, T min_value, T max_value, T* out) {
inline void random_numbers(int n, uint32_t seed, T min_value, T max_value, T* out) {
std::default_random_engine gen(seed);
std::uniform_int_distribution<T> d(min_value, max_value);
for (int i = 0; i < n; ++i) {
@@ -158,7 +116,8 @@ void random_numbers(int n, uint32_t seed, T min_value, T max_value, T* out) {
}
template <>
void random_numbers(int n, uint32_t seed, float min_value, float max_value, float* out) {
inline void random_numbers(int n, uint32_t seed, float min_value, float max_value,
float* out) {
std::default_random_engine gen(seed);
std::uniform_real_distribution<float> d(min_value, max_value);
for (int i = 0; i < n; ++i) {
@@ -167,8 +126,8 @@ void random_numbers(int n, uint32_t seed, float min_value, float max_value, floa
}
template <>
void random_numbers(int n, uint32_t seed, double min_value, double max_value,
double* out) {
inline void random_numbers(int n, uint32_t seed, double min_value, double max_value,
double* out) {
std::default_random_engine gen(seed);
std::uniform_real_distribution<double> d(min_value, max_value);
for (int i = 0; i < n; ++i) {
@@ -177,47 +136,14 @@ void random_numbers(int n, uint32_t seed, double min_value, double max_value,
}
void random_Int96_numbers(int n, uint32_t seed, int32_t min_value, int32_t max_value,
Int96* out) {
std::default_random_engine gen(seed);
std::uniform_int_distribution<int32_t> d(min_value, max_value);
for (int i = 0; i < n; ++i) {
out[i].value[0] = d(gen);
out[i].value[1] = d(gen);
out[i].value[2] = d(gen);
}
}
Int96* out);
void random_fixed_byte_array(int n, uint32_t seed, uint8_t* buf, int len, FLBA* out) {
std::default_random_engine gen(seed);
std::uniform_int_distribution<int> d(0, 255);
for (int i = 0; i < n; ++i) {
out[i].ptr = buf;
for (int j = 0; j < len; ++j) {
buf[j] = static_cast<uint8_t>(d(gen));
}
buf += len;
}
}
void random_fixed_byte_array(int n, uint32_t seed, uint8_t* buf, int len, FLBA* out);
void random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, int min_size,
int max_size) {
std::default_random_engine gen(seed);
std::uniform_int_distribution<int> d1(min_size, max_size);
std::uniform_int_distribution<int> d2(0, 255);
for (int i = 0; i < n; ++i) {
int len = d1(gen);
out[i].len = len;
out[i].ptr = buf;
for (int j = 0; j < len; ++j) {
buf[j] = static_cast<uint8_t>(d2(gen));
}
buf += len;
}
}
int max_size);
void random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, int max_size) {
random_byte_array(n, seed, buf, out, 0, max_size);
}
void random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, int max_size);
template <typename Type, typename Sequence>
std::shared_ptr<Buffer> EncodeValues(Encoding::type encoding, bool use_dictionary,
@@ -368,9 +294,9 @@ class DataPageBuilder {
};
template <>
void DataPageBuilder<BooleanType>::AppendValues(const ColumnDescriptor* d,
const std::vector<bool>& values,
Encoding::type encoding) {
inline void DataPageBuilder<BooleanType>::AppendValues(const ColumnDescriptor* d,
const std::vector<bool>& values,
Encoding::type encoding) {
if (encoding != Encoding::PLAIN) {
ParquetException::NYI("only plain encoding currently implemented");
}
@@ -463,25 +389,26 @@ class DictionaryPageBuilder {
};
template <>
DictionaryPageBuilder<BooleanType>::DictionaryPageBuilder(const ColumnDescriptor* d) {
inline DictionaryPageBuilder<BooleanType>::DictionaryPageBuilder(
const ColumnDescriptor* d) {
ParquetException::NYI("only plain encoding currently implemented for boolean");
}
template <>
std::shared_ptr<Buffer> DictionaryPageBuilder<BooleanType>::WriteDict() {
inline std::shared_ptr<Buffer> DictionaryPageBuilder<BooleanType>::WriteDict() {
ParquetException::NYI("only plain encoding currently implemented for boolean");
return nullptr;
}
template <>
std::shared_ptr<Buffer> DictionaryPageBuilder<BooleanType>::AppendValues(
inline std::shared_ptr<Buffer> DictionaryPageBuilder<BooleanType>::AppendValues(
const std::vector<TC>& values) {
ParquetException::NYI("only plain encoding currently implemented for boolean");
return nullptr;
}
template <typename Type>
static std::shared_ptr<DictionaryPage> MakeDictPage(
inline static std::shared_ptr<DictionaryPage> MakeDictPage(
const ColumnDescriptor* d, const std::vector<typename Type::c_type>& values,
const std::vector<int>& values_per_page, Encoding::type encoding,
std::vector<std::shared_ptr<Buffer>>& rle_indices) {
@@ -503,13 +430,15 @@ static std::shared_ptr<DictionaryPage> MakeDictPage(
// Given def/rep levels and values create multiple dict pages
template <typename Type>
static void PaginateDict(const ColumnDescriptor* d,
const std::vector<typename Type::c_type>& values,
const std::vector<int16_t>& def_levels, int16_t max_def_level,
const std::vector<int16_t>& rep_levels, int16_t max_rep_level,
int num_levels_per_page, const std::vector<int>& values_per_page,
std::vector<std::shared_ptr<Page>>& pages,
Encoding::type encoding = Encoding::RLE_DICTIONARY) {
inline static void PaginateDict(const ColumnDescriptor* d,
const std::vector<typename Type::c_type>& values,
const std::vector<int16_t>& def_levels,
int16_t max_def_level,
const std::vector<int16_t>& rep_levels,
int16_t max_rep_level, int num_levels_per_page,
const std::vector<int>& values_per_page,
std::vector<std::shared_ptr<Page>>& pages,
Encoding::type encoding = Encoding::RLE_DICTIONARY) {
int num_pages = static_cast<int>(values_per_page.size());
std::vector<std::shared_ptr<Buffer>> rle_indices;
std::shared_ptr<DictionaryPage> dict_page =
@@ -539,14 +468,15 @@ static void PaginateDict(const ColumnDescriptor* d,
// Given def/rep levels and values create multiple plain pages
template <typename Type>
static void PaginatePlain(const ColumnDescriptor* d,
const std::vector<typename Type::c_type>& values,
const std::vector<int16_t>& def_levels, int16_t max_def_level,
const std::vector<int16_t>& rep_levels, int16_t max_rep_level,
int num_levels_per_page,
const std::vector<int>& values_per_page,
std::vector<std::shared_ptr<Page>>& pages,
Encoding::type encoding = Encoding::PLAIN) {
static inline void PaginatePlain(const ColumnDescriptor* d,
const std::vector<typename Type::c_type>& values,
const std::vector<int16_t>& def_levels,
int16_t max_def_level,
const std::vector<int16_t>& rep_levels,
int16_t max_rep_level, int num_levels_per_page,
const std::vector<int>& values_per_page,
std::vector<std::shared_ptr<Page>>& pages,
Encoding::type encoding = Encoding::PLAIN) {
int num_pages = static_cast<int>(values_per_page.size());
int def_level_start = 0;
int def_level_end = 0;
@@ -574,12 +504,13 @@ static void PaginatePlain(const ColumnDescriptor* d,
// Generates pages from randomly generated data
template <typename Type>
static int MakePages(const ColumnDescriptor* d, int num_pages, int levels_per_page,
std::vector<int16_t>& def_levels, std::vector<int16_t>& rep_levels,
std::vector<typename Type::c_type>& values,
std::vector<uint8_t>& buffer,
std::vector<std::shared_ptr<Page>>& pages,
Encoding::type encoding = Encoding::PLAIN) {
static inline int MakePages(const ColumnDescriptor* d, int num_pages, int levels_per_page,
std::vector<int16_t>& def_levels,
std::vector<int16_t>& rep_levels,
std::vector<typename Type::c_type>& values,
std::vector<uint8_t>& buffer,
std::vector<std::shared_ptr<Page>>& pages,
Encoding::type encoding = Encoding::PLAIN) {
int num_levels = levels_per_page * num_pages;
int num_values = 0;
uint32_t seed = 0;
@@ -638,7 +569,7 @@ void inline InitValues<bool>(int num_values, std::vector<bool>& values,
}
template <>
void inline InitValues<ByteArray>(int num_values, std::vector<ByteArray>& values,
inline void InitValues<ByteArray>(int num_values, std::vector<ByteArray>& values,
std::vector<uint8_t>& buffer) {
int max_byte_array_len = 12;
int num_bytes = static_cast<int>(max_byte_array_len + sizeof(uint32_t));
@@ -647,7 +578,7 @@ void inline InitValues<ByteArray>(int num_values, std::vector<ByteArray>& values
random_byte_array(num_values, 0, buffer.data(), values.data(), max_byte_array_len);
}
void inline InitWideByteArrayValues(int num_values, std::vector<ByteArray>& values,
inline void InitWideByteArrayValues(int num_values, std::vector<ByteArray>& values,
std::vector<uint8_t>& buffer, int min_len,
int max_len) {
int num_bytes = static_cast<int>(max_len + sizeof(uint32_t));
@@ -657,7 +588,7 @@ void inline InitWideByteArrayValues(int num_values, std::vector<ByteArray>& valu
}
template <>
void inline InitValues<FLBA>(int num_values, std::vector<FLBA>& values,
inline void InitValues<FLBA>(int num_values, std::vector<FLBA>& values,
std::vector<uint8_t>& buffer) {
size_t nbytes = num_values * FLBA_LENGTH;
buffer.resize(nbytes);
@@ -665,7 +596,7 @@ void inline InitValues<FLBA>(int num_values, std::vector<FLBA>& values,
}
template <>
void inline InitValues<Int96>(int num_values, std::vector<Int96>& values,
inline void InitValues<Int96>(int num_values, std::vector<Int96>& values,
std::vector<uint8_t>& buffer) {
random_Int96_numbers(num_values, 0, std::numeric_limits<int32_t>::min(),
std::numeric_limits<int32_t>::max(), values.data());
@@ -720,10 +651,10 @@ class PrimitiveTypedTest : public ::testing::Test {
};
template <typename TestType>
void PrimitiveTypedTest<TestType>::SyncValuesOut() {}
inline void PrimitiveTypedTest<TestType>::SyncValuesOut() {}
template <>
void PrimitiveTypedTest<BooleanType>::SyncValuesOut() {
inline void PrimitiveTypedTest<BooleanType>::SyncValuesOut() {
std::vector<uint8_t>::const_iterator source_iterator = bool_buffer_out_.begin();
std::vector<T>::iterator destination_iterator = values_out_.begin();
while (source_iterator != bool_buffer_out_.end()) {
@@ -732,14 +663,14 @@ void PrimitiveTypedTest<BooleanType>::SyncValuesOut() {
}
template <typename TestType>
void PrimitiveTypedTest<TestType>::SetupValuesOut(int64_t num_values) {
inline void PrimitiveTypedTest<TestType>::SetupValuesOut(int64_t num_values) {
values_out_.clear();
values_out_.resize(num_values);
values_out_ptr_ = values_out_.data();
}
template <>
void PrimitiveTypedTest<BooleanType>::SetupValuesOut(int64_t num_values) {
inline void PrimitiveTypedTest<BooleanType>::SetupValuesOut(int64_t num_values) {
values_out_.clear();
values_out_.resize(num_values);
@@ -752,7 +683,7 @@ void PrimitiveTypedTest<BooleanType>::SetupValuesOut(int64_t num_values) {
}
template <typename TestType>
void PrimitiveTypedTest<TestType>::GenerateData(int64_t num_values) {
inline void PrimitiveTypedTest<TestType>::GenerateData(int64_t num_values) {
def_levels_.resize(num_values);
values_.resize(num_values);
@@ -763,7 +694,7 @@ void PrimitiveTypedTest<TestType>::GenerateData(int64_t num_values) {
}
template <>
void PrimitiveTypedTest<BooleanType>::GenerateData(int64_t num_values) {
inline void PrimitiveTypedTest<BooleanType>::GenerateData(int64_t num_values) {
def_levels_.resize(num_values);
values_.resize(num_values);