mirrors/arrow
0
0 Fork 0
mirror of https://github.com/apache/arrow.git synced 2026-10-28 16:47:19 +08:00
Code Activity

Add integration tests for the functions over strings

Browse Source
This commit is contained in:
Anthony Louis committed 2021-04-09 19:34:23 -03:00
1 parent d836d27c05
commit 8e2cfee217
4 files changed
+67 -79
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cpp/src/gandiva/gdv_function_stubs.cc
+15 -11
View File
@@ -145,11 +145,11 @@ int32_t gdv_fn_populate_varlen_vector(int64_t context_ptr, int8_t* data_ptr,
#define SHA256_HASH_FUNCTION_BUF(TYPE) \
GANDIVA_EXPORT \
const char *gdv_fn_hash_sha256_from_##TYPE(int64_t context, \
gdv_##TYPE value, \
uint64_t value_length, \
bool value_validity, \
uint32_t *out_length) { \
const char *gdv_fn_sha256_from_##TYPE(int64_t context, \
gdv_##TYPE value, \
int32_t value_length, \
bool value_validity, \
u_int32_t *out_length) { \
if(!value_validity){ \
*out_length = 0; \
return ""; \
@@ -586,23 +586,27 @@ void ExportedStubFunctions::AddMappings(Engine* engine) const {
args = {
types->i64_type(), // context
types->i8_ptr_type(), // const char*
types->i64_type() // value_length
types->i32_type(), // value_length
types->i1_type(), // validity
types->i32_ptr_type() // out
};
engine->AddGlobalMappingForFunc("gdv_fn_hash_sha256_from_utf8",
engine->AddGlobalMappingForFunc("gdv_fn_sha256_from_utf8",
types->i8_ptr_type() /*return_type*/, args,
reinterpret_cast<void *>(gdv_fn_hash_sha256_from_utf8));
reinterpret_cast<void *>(gdv_fn_sha256_from_utf8));
// gdv_fn_hash_sha256_from_binary
args = {
types->i64_type(), // context
types->i8_ptr_type(), // const char*
types->i64_type() // value_length
types->i32_type(), // value_length
types->i1_type(), // validity
types->i32_ptr_type() // out
};
engine->AddGlobalMappingForFunc("gdv_fn_hash_sha256_from_binary",
engine->AddGlobalMappingForFunc("gdv_fn_sha256_from_binary",
types->i8_ptr_type() /*return_type*/, args,
reinterpret_cast<void *>(gdv_fn_hash_sha256_from_binary));
reinterpret_cast<void *>(gdv_fn_sha256_from_binary));
}
cpp/src/gandiva/gdv_function_stubs.h
+5 -5
View File
@@ -69,11 +69,11 @@ int gdv_fn_time_with_zone(int* time_fields, const char* zone, int zone_len,
const char *gdv_fn_sha256_from_float64(int64_t context, double value, bool validity, unsigned int *out_length);
const char *gdv_fn_hash_sha256_from_string(int64_t context,
const char *value,
uint64_t value_length,
bool value_validity,
uint32_t *out_length);
const char *gdv_fn_sha256_from_utf8(int64_t context,
char *value,
int32_t value_length,
bool value_validity,
uint32_t *out_length);
const char * gdv_fn_sha128_from_numeric(int64_t context, double value);
cpp/src/gandiva/gdv_function_stubs_test.cc
+1 -63
View File
@@ -160,68 +160,6 @@ TEST(TestGdvFnStubs, TestCastFloat8) {
ctx.Reset();
}
TEST(TestGdvFnStubs, TestSha256Numeric) {
gandiva::ExecutionContext ctx;
int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
// test zero boundaries
const char *double_zero_hash = gdv_fn_sha256_from_numeric(ctx_ptr, 0.0);
const char *zero_hash = gdv_fn_sha256_from_numeric(ctx_ptr, 0);
const char *zero_one_hash = gdv_fn_sha256_from_numeric(ctx_ptr, 0.1);
EXPECT_STRNE(zero_hash, zero_one_hash);
EXPECT_STREQ(double_zero_hash,zero_hash);
// tests minus zero cases
const char *double_minus_zero_hash = gdv_fn_sha256_from_numeric(ctx_ptr, -0.0);
const char *minus_zero_hash = gdv_fn_sha256_from_numeric(ctx_ptr, -0);
EXPECT_STREQ(zero_hash, minus_zero_hash);
EXPECT_STREQ(zero_hash, double_minus_zero_hash);
ctx.Reset();
}
TEST(TestGdvFnStubs, TestSha256String) {
gandiva::ExecutionContext ctx;
int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
const char *zero_hash = gdv_fn_sha256_from_numeric(ctx_ptr, 0.0);
const char *test_string = "ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeıʃn\n"
"Y [ˈʏpsilɔn], Yen [jɛn], Yoga [ˈjoːgɑ]";
const char *first_sha_hash = gdv_fn_hash_sha256_from_string(ctx_ptr, test_string, strlen(test_string));
const char *expected_first_hash ="55aeb2e789871dbd289edae94d4c1c82a1c25ca0bcd5a873924da2fefdd57acb";
EXPECT_STRNE(first_sha_hash, zero_hash);
EXPECT_STREQ(expected_first_hash, first_sha_hash);
const char *test_string_modified = "ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeın\n"
"Y [ˈʏpsilɔn], Yen [jɛn], Yoga [ˈjoːgɑ] コンニチハ";
const char
*second_sha_hash = gdv_fn_hash_sha256_from_string(ctx_ptr, test_string_modified, strlen(test_string_modified));
const char *expected_second_hash ="86b29c13d0d0e26ea8f85bfa649dc9b8622ae59a4da2409d7d9b463e86e796f2";
EXPECT_STREQ(expected_second_hash,second_sha_hash);
EXPECT_STRNE(second_sha_hash, first_sha_hash);
const char *empty_string_hash = gdv_fn_hash_sha256_from_string(ctx_ptr, "", strlen(""));
EXPECT_STREQ("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",empty_string_hash);
gdv_fn_hash_sha256_from_string(ctx_ptr, nullptr, 0);
EXPECT_THAT(ctx.get_error(),
::testing::HasSubstr("A null value was given to be hashed."));
ctx.Reset();
}
TEST(TestGdvFnStubs, TestSha128Numeric) {
gandiva::ExecutionContext ctx;
@@ -273,7 +211,7 @@ TEST(TestGdvFnStubs, TestSha128String) {
EXPECT_STREQ("da39a3ee5e6b4b0d3255bfef95601890afd80709",empty_string_hash);
gdv_fn_hash_sha256_from_string(ctx_ptr, nullptr, 0);
gdv_fn_hash_sha128_from_string(ctx_ptr, nullptr, 0);
EXPECT_THAT(ctx.get_error(),
::testing::HasSubstr("A null value was given to be hashed."));
cpp/src/gandiva/tests/hash_test.cc
+46
View File
@@ -219,4 +219,50 @@ TEST_F(TestHash, TestSha256Simple) {
EXPECT_ARROW_ARRAY_EQUALS(outputs.at(2), outputs.at(3));
}
TEST_F(TestHash, TestSha256Varlen) {
// schema for input fields
auto field_a = field("a", utf8());
auto schema = arrow::schema({field_a});
// output fields
auto res_0 = field("res0", utf8());
// build expressions.
// hashSHA256(a)
auto node_a = TreeExprBuilder::MakeField(field_a);
auto hashSha256 = TreeExprBuilder::MakeFunction("hashSHA256", {node_a}, utf8());
auto expr_0 = TreeExprBuilder::MakeExpression(hashSha256, res_0);
// Build a projector for the expressions.
std::shared_ptr<Projector> projector;
auto status =
Projector::Make(schema, {expr_0}, TestConfiguration(), &projector);
EXPECT_TRUE(status.ok()) << status.message();
// Create a row-batch with some sample data
int num_records = 3;
std::string first_string = "ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeıʃn\nY [ˈʏpsilɔn], Yen [jɛn], Yoga [ˈjoːgɑ]";
std::string second_string = "ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeın\nY [ˈʏpsilɔn], Yen [jɛn], Yoga [ˈjoːgɑ] コンニチハ";
auto array_a =
MakeArrowArrayUtf8({"foo", first_string, second_string}, {false, true, true});
// prepare input record batch
auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array_a});
// Evaluate expression
arrow::ArrayVector outputs;
status = projector->Evaluate(*in_batch, pool_, &outputs);
EXPECT_TRUE(status.ok());
auto response = outputs.at(0);
EXPECT_EQ(response->null_count(), 0);
EXPECT_EQ(response->GetScalar(0).ValueOrDie()->ToString(), "");
for (int i = 1; i < num_records; ++i) {
const auto &value_at_position = response->GetScalar(i).ValueOrDie()->ToString();
EXPECT_NE(value_at_position, response->GetScalar(i - 1).ValueOrDie()->ToString());
}
}
} // namespace gandiva