Add base tests for hash 128 gdv function

This commit is contained in:
João Pedro authored and Anthony Louis committed 2021-04-09 19:34:23 -03:00
1 parent d05681a092
commit 3f4d0fa3e5
1 file changed
+114 -3
+114 -3
View File
@@ -152,7 +152,7 @@ TEST_F(TestHash, TestBuf) {
TEST_F(TestHash, TestSha256Simple) {
// schema for input fields
auto field_a = field("a", int32());
auto field_b = field("b", int64());\
auto field_b = field("b", int64());
auto field_c = field("c", float32());
auto field_d = field("d", float64());
auto schema = arrow::schema({field_a, field_b, field_c, field_d});
@@ -187,8 +187,6 @@ TEST_F(TestHash, TestSha256Simple) {
Projector::Make(schema, {expr_0, expr_1, expr_2, expr_3}, TestConfiguration(), &projector);
EXPECT_TRUE(status.ok()) << status.message();
// Create a row-batch with some sample data
int num_records = 2;
auto validity_array = {false, true};
@@ -265,4 +263,117 @@ TEST_F(TestHash, TestSha256Varlen) {
}
}
TEST_F(TestHash, TestSha128Simple) {
// schema for input fields
auto field_a = field("a", int32());
auto field_b = field("b", int64());
auto field_c = field("c", float32());
auto field_d = field("d", float64());
auto schema = arrow::schema({field_a, field_b, field_c, field_d});
// output fields
auto res_0 = field("res0", utf8());
auto res_1 = field("res1", utf8());
auto res_2 = field("res2", utf8());
auto res_3 = field("res3", utf8());
// build expressions.
// hashSHA128(a)
auto node_a = TreeExprBuilder::MakeField(field_a);
auto hashSha128_1 = TreeExprBuilder::MakeFunction("hashSHA128", {node_a}, utf8());
auto expr_0 = TreeExprBuilder::MakeExpression(hashSha128_1, res_0);
auto node_b = TreeExprBuilder::MakeField(field_b);
auto hashSha128_2 = TreeExprBuilder::MakeFunction("hashSHA128", {node_b}, utf8());
auto expr_1 = TreeExprBuilder::MakeExpression(hashSha128_2, res_1);
auto node_c = TreeExprBuilder::MakeField(field_c);
auto hashSha128_3 = TreeExprBuilder::MakeFunction("hashSHA128", {node_c}, utf8());
auto expr_2 = TreeExprBuilder::MakeExpression(hashSha128_3, res_2);
auto node_d = TreeExprBuilder::MakeField(field_d);
auto hashSha128_4 = TreeExprBuilder::MakeFunction("hashSHA128", {node_d}, utf8());
auto expr_3 = TreeExprBuilder::MakeExpression(hashSha128_4, res_3);
// Build a projector for the expressions.
std::shared_ptr<Projector> projector;
auto status =
Projector::Make(schema, {expr_0, expr_1, expr_2, expr_3}, TestConfiguration(), &projector);
EXPECT_TRUE(status.ok()) << status.message();
// Create a row-batch with some sample data
int num_records = 2;
auto validity_array = {false, true};
auto array_int32 =
MakeArrowArrayInt32({1, 0}, validity_array);
auto array_int64 =
MakeArrowArrayInt64({1, 0}, validity_array);
auto array_float32 =
MakeArrowArrayFloat32({1.0, 0.0}, validity_array);
auto array_float64 =
MakeArrowArrayFloat64({1.0, 0.0}, validity_array);
// prepare input record batch
auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array_int32, array_int64,
array_float32, array_float64});
// Evaluate expression
arrow::ArrayVector outputs;
status = projector->Evaluate(*in_batch, pool_, &outputs);
EXPECT_TRUE(status.ok());
EXPECT_ARROW_ARRAY_EQUALS(outputs.at(0), outputs.at(1));
EXPECT_ARROW_ARRAY_EQUALS(outputs.at(1), outputs.at(2));
EXPECT_ARROW_ARRAY_EQUALS(outputs.at(2), outputs.at(3));
}
TEST_F(TestHash, TestSha128Varlen) {
// schema for input fields
auto field_a = field("a", utf8());
auto schema = arrow::schema({field_a});
// output fields
auto res_0 = field("res0", utf8());
// build expressions.
// hashSHA128(a)
auto node_a = TreeExprBuilder::MakeField(field_a);
auto hashSha128 = TreeExprBuilder::MakeFunction("hashSHA128", {node_a}, utf8());
auto expr_0 = TreeExprBuilder::MakeExpression(hashSha128, res_0);
// Build a projector for the expressions.
std::shared_ptr<Projector> projector;
auto status =
Projector::Make(schema, {expr_0}, TestConfiguration(), &projector);
EXPECT_TRUE(status.ok()) << status.message();
// Create a row-batch with some sample data
int num_records = 3;
std::string first_string = "ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeıʃn\nY [ˈʏpsilɔn], Yen [jɛn], Yoga [ˈjoːgɑ]";
std::string second_string = "ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeın\nY [ˈʏpsilɔn], Yen [jɛn], Yoga [ˈjoːgɑ] コンニチハ";
auto array_a =
MakeArrowArrayUtf8({"foo", first_string, second_string}, {false, true, true});
// prepare input record batch
auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array_a});
// Evaluate expression
arrow::ArrayVector outputs;
status = projector->Evaluate(*in_batch, pool_, &outputs);
EXPECT_TRUE(status.ok());
auto response = outputs.at(0);
EXPECT_EQ(response->null_count(), 0);
EXPECT_EQ(response->GetScalar(0).ValueOrDie()->ToString(), "");
for (int i = 1; i < num_records; ++i) {
const auto &value_at_position = response->GetScalar(i).ValueOrDie()->ToString();
EXPECT_NE(value_at_position, response->GetScalar(i - 1).ValueOrDie()->ToString());
}
}
} // namespace gandiva