mirrors/arrow
0
0 Fork 0
mirror of https://github.com/apache/arrow.git synced 2026-10-28 16:47:19 +08:00
Code Activity

Add macros for all types for sha256

Browse Source
This commit is contained in:
Anthony Louis committed 2021-04-09 19:34:23 -03:00
1 parent 28fb105820
commit d836d27c05
5 files changed
+84 -44
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cpp/src/gandiva/function_registry_common.h
+10
View File
@@ -208,6 +208,16 @@ typedef std::unordered_map<const FunctionSignature*, const NativeFunction*, KeyH
DataTypeVector{TYPE(), int64()}, int64(), kResultNullNever, \
ARROW_STRINGIFY(NAME##WithSeed_##TYPE))
// HashSHA256 functions that :
// - NULL handling is of type NULL_NEVER
//
// The function name includes the base name & input type name. gdv_fn_sha256_from_float64
#define HASH_SHA256_SAFE_NULL_NEVER(NAME, ALIASES, TYPE) \
NativeFunction(#NAME, std::vector<std::string> ALIASES, \
DataTypeVector{TYPE()}, utf8(), kResultNullNever, \
ARROW_STRINGIFY(gdv_fn_sha256_from_##TYPE), \
NativeFunction::kNeedsContext)
// Iterate the inner macro over all numeric types
#define NUMERIC_TYPES(INNER, NAME, ALIASES) \
INNER(NAME, ALIASES, int8), INNER(NAME, ALIASES, int16), INNER(NAME, ALIASES, int32), \
cpp/src/gandiva/function_registry_hash.cc
+6 -1
View File
@@ -33,6 +33,9 @@ namespace gandiva {
#define HASH64_SEED_SAFE_NULL_NEVER_FN(name, ALIASES) \
NUMERIC_BOOL_DATE_VAR_LEN_TYPES(HASH64_SEED_SAFE_NULL_NEVER, name, ALIASES)
#define HASH_SHA256_SAFE_NULL_NEVER_FN(name, ALIASES) \
NUMERIC_BOOL_DATE_VAR_LEN_TYPES(HASH_SHA256_SAFE_NULL_NEVER, name, ALIASES)
std::vector<NativeFunction> GetHashFunctionRegistry() {
static std::vector<NativeFunction> hash_fn_registry_ = {
HASH32_SAFE_NULL_NEVER_FN(hash, {}),
@@ -46,7 +49,9 @@ std::vector<NativeFunction> GetHashFunctionRegistry() {
HASH64_SAFE_NULL_NEVER_FN(hash64AsDouble, {}),
HASH64_SEED_SAFE_NULL_NEVER_FN(hash64, {}),
HASH64_SEED_SAFE_NULL_NEVER_FN(hash64AsDouble, {})};
HASH64_SEED_SAFE_NULL_NEVER_FN(hash64AsDouble, {}),
HASH_SHA256_SAFE_NULL_NEVER_FN(hashSHA256, {})};
return hash_fn_registry_;
}
cpp/src/gandiva/gdv_function_stubs.cc
+57 -36
View File
@@ -30,24 +30,6 @@
#include "gandiva/to_date_holder.h"
// Expand inner macro for all numeric types.
#define NUMERIC_BOOL_DATE_TYPES(INNER) \
INNER(int8) \
INNER(int16) \
INNER(int32) \
INNER(int64) \
INNER(uint8) \
INNER(uint16) \
INNER(uint32) \
INNER(uint64) \
INNER(float32) \
INNER(float64) \
INNER(boolean) \
INNER(date64) \
INNER(date32) \
INNER(time32) \
INNER(timestamp)
/// Stub functions that can be accessed from LLVM or the pre-compiled library.
extern "C" {
@@ -151,30 +133,31 @@ int32_t gdv_fn_populate_varlen_vector(int64_t context_ptr, int8_t* data_ptr,
return ""; \
} \
\
double value_as_double = (double)value; \
auto value_as_long = gandiva::HashUtils::double_to_long((double)value); \
\
const char *result = gandiva::HashUtils::hash_using_SHA256(context, \
&value, \
sizeof(value_as_double), \
&value_as_long, \
sizeof(value_as_long), \
out_length); \
\
return result; \
} \
GANDIVA_EXPORT
const char *gdv_fn_hash_sha256_from_string(int64_t context,
const char *value,
uint64_t value_length,
bool value_validity,
uint32_t *out_length) {
if(!value_validity){
*out_length = 0;
return "";
#define SHA256_HASH_FUNCTION_BUF(TYPE) \
GANDIVA_EXPORT \
const char *gdv_fn_hash_sha256_from_##TYPE(int64_t context, \
gdv_##TYPE value, \
uint64_t value_length, \
bool value_validity, \
uint32_t *out_length) { \
if(!value_validity){ \
*out_length = 0; \
return ""; \
} \
\
return gandiva::HashUtils::hash_using_SHA256(context, value, value_length, out_length); \
}
return gandiva::HashUtils::hash_using_SHA256(context, value, value_length, out_length);
}
GANDIVA_EXPORT
const char* gdv_fn_sha128_from_numeric(int64_t context, double value){
uint64_t value_as_long = gandiva::HashUtils::double_to_long(value);
@@ -187,9 +170,36 @@ const char* gdv_fn_hash_sha128_from_string(int64_t context, const char* value, u
return gandiva::HashUtils::hash_using_SHA128(context, value, value_length, nullptr);
}
// Expand inner macro for all numeric types.
#define NUMERIC_BOOL_DATE_TYPES(INNER) \
INNER(int8) \
INNER(int16) \
INNER(int32) \
INNER(int64) \
INNER(uint8) \
INNER(uint16) \
INNER(uint32) \
INNER(uint64) \
INNER(float32) \
INNER(float64) \
INNER(boolean) \
INNER(date64) \
INNER(date32) \
INNER(time32) \
INNER(timestamp)
// Expand inner macro for all numeric types.
#define VAR_LEN_TYPES(INNER) \
INNER(utf8) \
INNER(binary)
NUMERIC_BOOL_DATE_TYPES(SHA256_HASH_FUNCTION)
VAR_LEN_TYPES(SHA256_HASH_FUNCTION_BUF)
#undef NUMERIC_BOOL_DATE_TYPES
#undef VAR_LEN_TYPES
int32_t gdv_fn_dec_from_string(int64_t context, const char* in, int32_t in_length,
int32_t* precision_from_str, int32_t* scale_from_str,
@@ -572,16 +582,27 @@ void ExportedStubFunctions::AddMappings(Engine* engine) const {
types->i8_ptr_type() /*return_type*/, args,
reinterpret_cast<void *>(gdv_fn_sha256_from_timestamp));
// gdv_fn_hash_sha256_from_string
// gdv_fn_hash_sha256_from_utf8
args = {
types->i64_type(), // context
types->i8_ptr_type(), // const char*
types->i64_type() // value_length
};
engine->AddGlobalMappingForFunc("gdv_fn_hash_sha256_from_string",
engine->AddGlobalMappingForFunc("gdv_fn_hash_sha256_from_utf8",
types->i8_ptr_type() /*return_type*/, args,
reinterpret_cast<void *>(gdv_fn_hash_sha256_from_string));
reinterpret_cast<void *>(gdv_fn_hash_sha256_from_utf8));
// gdv_fn_hash_sha256_from_binary
args = {
types->i64_type(), // context
types->i8_ptr_type(), // const char*
types->i64_type() // value_length
};
engine->AddGlobalMappingForFunc("gdv_fn_hash_sha256_from_binary",
types->i8_ptr_type() /*return_type*/, args,
reinterpret_cast<void *>(gdv_fn_hash_sha256_from_binary));
}
cpp/src/gandiva/precompiled/hash.cc
+1 -1
View File
@@ -379,7 +379,7 @@ FORCE_INLINE gdv_int32 hash32_buf(const gdv_uint8* buf, int len, gdv_int32 seed)
return is_valid ? hash32_buf(reinterpret_cast<const uint8_t*>(in), len, 0) : 0; \
}
// Expand inner macro for all numeric types.
// Expand inner macro for all non-numeric types.
#define VAR_LEN_TYPES(INNER, NAME) \
INNER(NAME, utf8) \
INNER(NAME, binary)
cpp/src/gandiva/tests/hash_test.cc
+10 -6
View File
@@ -190,20 +190,20 @@ TEST_F(TestHash, TestSha256Simple) {
// Create a row-batch with some sample data
int num_records = 4;
auto validity_array = {false, true, true, true};
int num_records = 2;
auto validity_array = {false, true};
auto array_int32 =
MakeArrowArrayInt32({1, 2, 3, 4}, validity_array);
MakeArrowArrayInt32({1, 0}, validity_array);
auto array_int64 =
MakeArrowArrayInt64({1, 2, 3, 4}, validity_array);
MakeArrowArrayInt64({1, 0}, validity_array);
auto array_float32 =
MakeArrowArrayFloat32({1.0, 2.0, 3.0, 4.0}, validity_array);
MakeArrowArrayFloat32({1.0, 0.0}, validity_array);
auto array_float64 =
MakeArrowArrayFloat64({1.0, 2.0, 3.0, 4.0}, validity_array);
MakeArrowArrayFloat64({1.0, 0.0}, validity_array);
// prepare input record batch
auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array_int32, array_int64,
@@ -213,6 +213,10 @@ TEST_F(TestHash, TestSha256Simple) {
arrow::ArrayVector outputs;
status = projector->Evaluate(*in_batch, pool_, &outputs);
EXPECT_TRUE(status.ok());
EXPECT_ARROW_ARRAY_EQUALS(outputs.at(0), outputs.at(1));
EXPECT_ARROW_ARRAY_EQUALS(outputs.at(1), outputs.at(2));
EXPECT_ARROW_ARRAY_EQUALS(outputs.at(2), outputs.at(3));
}
} // namespace gandiva