mirror of
https://codeberg.org/ziglang/zig.git
synced 2026-10-28 10:45:48 +08:00
zig cc: update intrinsic headers to LLVM 23
This commit is contained in:
56 files changed
+6963
-4000
No files matched your search
+170
@@ -0,0 +1,170 @@
|
||||
/*===-- __clang_spirv_libdevice_declares.h - decls for libdevice functions --===
|
||||
*
|
||||
* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
* See https://llvm.org/LICENSE.txt for license information.
|
||||
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
*
|
||||
*===-----------------------------------------------------------------------===
|
||||
*/
|
||||
|
||||
#ifndef __CLANG_SPIRV_LIBDEVICE_DECLARES_H__
|
||||
#define __CLANG_SPIRV_LIBDEVICE_DECLARES_H__
|
||||
|
||||
#if defined(__cplusplus)
|
||||
extern "C" {
|
||||
#else
|
||||
_Pragma("push_macro(\"bool\")");
|
||||
#define bool _Bool
|
||||
#endif
|
||||
|
||||
#define _CLC_OVERLOAD [[clang::overloadable]]
|
||||
#define _CLC_CONSTFN [[gnu::const]]
|
||||
// TODO: Add vector versions of the API in case it is required.
|
||||
_CLC_OVERLOAD _CLC_CONSTFN unsigned int __spirv_ocl_s_abs(int);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN unsigned long __spirv_ocl_s_abs(long);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN unsigned long long __spirv_ocl_s_abs(long long);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN int __spirv_ocl_s_max(int, int);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN long __spirv_ocl_s_max(long, long);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN long long __spirv_ocl_s_max(long long, long long);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN int __spirv_ocl_s_min(int, int);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN long __spirv_ocl_s_min(long, long);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN long long __spirv_ocl_s_min(long long, long long);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN unsigned int __spirv_ocl_u_max(unsigned int,
|
||||
unsigned int);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN unsigned long __spirv_ocl_u_max(unsigned long,
|
||||
unsigned long);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN unsigned long long
|
||||
__spirv_ocl_u_max(unsigned long long, unsigned long long);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN unsigned int __spirv_ocl_u_min(unsigned int,
|
||||
unsigned int);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN unsigned long __spirv_ocl_u_min(unsigned long,
|
||||
unsigned long);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN unsigned long long
|
||||
__spirv_ocl_u_min(unsigned long long, unsigned long long);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_acos(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_acos(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_acosh(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_acosh(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_asin(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_asin(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_asinh(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_asinh(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_atan(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_atan(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_atan2(float, float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_atan2(double, double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_atanh(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_atanh(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_cbrt(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_cbrt(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_ceil(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_ceil(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_cos(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_cos(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_cosh(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_cosh(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_cospi(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_cospi(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_erf(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_erf(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_erfc(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_erfc(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_exp(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_exp(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_exp2(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_exp2(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_exp10(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_exp10(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_expm1(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_expm1(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN bool __spirv_IsNan(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN bool __spirv_IsNan(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN bool __spirv_IsInf(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN bool __spirv_IsInf(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN bool __spirv_IsFinite(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN bool __spirv_IsFinite(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_copysign(float, float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_copysign(double, double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_ldexp(float, int);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_ldexp(double, int);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_fabs(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_fabs(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_logb(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_logb(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_fmax(float, float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_fmax(double, double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_fmin(float, float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_fmin(double, double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_fdim(float, float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_fdim(double, double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_floor(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_floor(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_fma(float, float, float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_fma(double, double, double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_fmod(float, float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_fmod(double, double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_frexp(float, int *);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_frexp(double, int *);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_hypot(float, float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_hypot(double, double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN int __spirv_ocl_ilogb(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN int __spirv_ocl_ilogb(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_lgamma(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_lgamma(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_round(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_round(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_log(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_log(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_log10(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_log10(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_log1p(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_log1p(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_log2(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_log2(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_modf(float, float *);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_modf(double, double *);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_nan(int);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_nan(unsigned int);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_nan(long);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_nan(unsigned long);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_nextafter(float, float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_nextafter(double, double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_sqrt(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_sqrt(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_rsqrt(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_rsqrt(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_pow(float, float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_pow(double, double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_pown(float, int);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_pown(double, int);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_remainder(float, float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_remainder(double, double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_remquo(float, float, int *);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_remquo(double, double, int *);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_sin(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_sin(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_sincos(float, float *);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_sincos(double, double *);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_sinh(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_sinh(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_sinpi(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_sinpi(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_tan(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_tan(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_tanh(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_tanh(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_tgamma(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_tgamma(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_trunc(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_trunc(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_rint(float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_rint(double);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_fclamp(float, float, float);
|
||||
_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_fclamp(double, double, double);
|
||||
|
||||
#if defined(__cplusplus)
|
||||
} // extern "C"
|
||||
#else
|
||||
_Pragma("pop_macro(\"bool\")");
|
||||
#endif
|
||||
#endif // __CLANG_SPIRV_LIBDEVICE_DECLARES_H__
|
||||
Vendored
+719
@@ -0,0 +1,719 @@
|
||||
/*===---- __clang_spirv_math.h - Device-side SPIRV math support ------------===
|
||||
*
|
||||
* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
* See https://llvm.org/LICENSE.txt for license information.
|
||||
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
*
|
||||
*===-----------------------------------------------------------------------===
|
||||
*/
|
||||
#ifndef __CLANG_SPIRV_MATH_H__
|
||||
#define __CLANG_SPIRV_MATH_H__
|
||||
|
||||
#if !defined(__SPIRV__) && !defined(__OPENMP_SPIRV__)
|
||||
#error "This file is for SPIRV and OpenMP SPIRV device compilation only."
|
||||
#endif
|
||||
|
||||
// The __CLANG_GPU_DISABLE_MATH_WRAPPERS macro provides a way to let standard
|
||||
// libcalls reach the link step instead of being eagerly replaced.
|
||||
#ifndef __CLANG_GPU_DISABLE_MATH_WRAPPERS
|
||||
|
||||
// __DEVICE__ is a helper macro with common set of attributes for the wrappers
|
||||
// we implement in this file. We need static in order to avoid emitting unused
|
||||
// functions and __forceinline__ helps inlining these wrappers at -O1.
|
||||
#pragma push_macro("__DEVICE__")
|
||||
#ifdef __OPENMP_SPIRV__
|
||||
#if defined(__cplusplus)
|
||||
#define __DEVICE__ static constexpr __attribute__((always_inline, nothrow))
|
||||
#else
|
||||
#define __DEVICE__ static __attribute__((always_inline, nothrow))
|
||||
#endif
|
||||
#else
|
||||
#define __DEVICE__ static __device__ __forceinline__
|
||||
#endif
|
||||
|
||||
__DEVICE__
|
||||
float __cosf(float __x) { return __spirv_ocl_cos(__x); }
|
||||
__DEVICE__
|
||||
float __exp10f(float __x) { return __spirv_ocl_exp10(__x); }
|
||||
__DEVICE__
|
||||
float __expf(float __x) { return __spirv_ocl_exp(__x); }
|
||||
|
||||
__DEVICE__
|
||||
float __fadd_rd(float __x, float __y) {
|
||||
float sum = __x + __y;
|
||||
float rounded = __spirv_ocl_floor(sum);
|
||||
if (rounded > sum)
|
||||
rounded -= 1.0f;
|
||||
return rounded;
|
||||
}
|
||||
|
||||
__DEVICE__
|
||||
float __fadd_rn(float __x, float __y) { return __spirv_ocl_rint(__x + __y); }
|
||||
__DEVICE__
|
||||
float __fadd_ru(float __x, float __y) { return __spirv_ocl_ceil(__x + __y); }
|
||||
__DEVICE__
|
||||
float __fadd_rz(float __x, float __y) { return __spirv_ocl_trunc(__x + __y); }
|
||||
|
||||
__DEVICE__
|
||||
float __fdiv_rd(float __x, float __y) {
|
||||
float res = __x / __y;
|
||||
float rounded = __spirv_ocl_floor(res);
|
||||
if (rounded > res)
|
||||
rounded -= 1.0f;
|
||||
return rounded;
|
||||
}
|
||||
__DEVICE__
|
||||
float __fdiv_rn(float __x, float __y) { return __spirv_ocl_rint(__x / __y); }
|
||||
__DEVICE__
|
||||
float __fdiv_ru(float __x, float __y) { return __spirv_ocl_ceil(__x / __y); }
|
||||
__DEVICE__
|
||||
float __fdiv_rz(float __x, float __y) { return __spirv_ocl_trunc(__x / __y); }
|
||||
__DEVICE__
|
||||
float __fdividef(float __x, float __y) { return __x / __y; }
|
||||
|
||||
__DEVICE__
|
||||
float __fmaf_rd(float __x, float __y, float __z) {
|
||||
float res = __x * __y + __z;
|
||||
float rounded = __spirv_ocl_floor(res);
|
||||
if (rounded > res)
|
||||
rounded -= 1.0f;
|
||||
return rounded;
|
||||
}
|
||||
__DEVICE__
|
||||
float __fmaf_rn(float __x, float __y, float __z) {
|
||||
return __spirv_ocl_rint(__x * __y + __z);
|
||||
}
|
||||
__DEVICE__
|
||||
float __fmaf_ru(float __x, float __y, float __z) {
|
||||
return __spirv_ocl_ceil(__x * __y + __z);
|
||||
}
|
||||
__DEVICE__
|
||||
float __fmaf_rz(float __x, float __y, float __z) {
|
||||
return __spirv_ocl_trunc(__x * __y + __z);
|
||||
}
|
||||
|
||||
__DEVICE__
|
||||
float __fmul_rd(float __x, float __y) {
|
||||
float res = __x * __y;
|
||||
float rounded = __spirv_ocl_floor(res);
|
||||
if (rounded > res)
|
||||
rounded -= 1.0f;
|
||||
return rounded;
|
||||
}
|
||||
__DEVICE__
|
||||
float __fmul_rn(float __x, float __y) { return __spirv_ocl_rint(__x * __y); }
|
||||
__DEVICE__
|
||||
float __fmul_ru(float __x, float __y) { return __spirv_ocl_ceil(__x * __y); }
|
||||
__DEVICE__
|
||||
float __fmul_rz(float __x, float __y) { return __spirv_ocl_trunc(__x * __y); }
|
||||
|
||||
__DEVICE__
|
||||
float __frcp_rd(float __x) { return __fdiv_rd(1.0f, __x); }
|
||||
__DEVICE__
|
||||
float __frcp_rn(float __x) { return __fdiv_rn(1.0f, __x); }
|
||||
__DEVICE__
|
||||
float __frcp_ru(float __x) { return __fdiv_ru(1.0f, __x); }
|
||||
__DEVICE__
|
||||
float __frcp_rz(float __x) { return __fdiv_rz(1.0f, __x); }
|
||||
__DEVICE__
|
||||
|
||||
float __frsqrt_rn(float __x) {
|
||||
return __spirv_ocl_rint(__spirv_ocl_rsqrt(__x));
|
||||
}
|
||||
|
||||
__DEVICE__
|
||||
float __fsqrt_rd(float __x) {
|
||||
float res = __spirv_ocl_sqrt(__x);
|
||||
float rounded = __spirv_ocl_floor(res);
|
||||
if (rounded > res)
|
||||
rounded -= 1.0f;
|
||||
return rounded;
|
||||
}
|
||||
__DEVICE__
|
||||
float __fsqrt_rn(float __x) { return __spirv_ocl_rint(__spirv_ocl_sqrt(__x)); }
|
||||
__DEVICE__
|
||||
float __fsqrt_ru(float __x) { return __spirv_ocl_ceil(__spirv_ocl_sqrt(__x)); }
|
||||
__DEVICE__
|
||||
float __fsqrt_rz(float __x) { return __spirv_ocl_trunc(__spirv_ocl_sqrt(__x)); }
|
||||
|
||||
__DEVICE__
|
||||
float __fsub_rd(float __x, float __y) {
|
||||
float res = __x - __y;
|
||||
float rounded = __spirv_ocl_floor(res);
|
||||
if (rounded > res)
|
||||
rounded -= 1.0f;
|
||||
return rounded;
|
||||
}
|
||||
__DEVICE__
|
||||
float __fsub_rn(float __x, float __y) { return __spirv_ocl_rint(__x - __y); }
|
||||
__DEVICE__
|
||||
float __fsub_ru(float __x, float __y) { return __spirv_ocl_ceil(__x - __y); }
|
||||
__DEVICE__
|
||||
float __fsub_rz(float __x, float __y) { return __spirv_ocl_trunc(__x - __y); }
|
||||
__DEVICE__
|
||||
float __log10f(float __x) { return __spirv_ocl_log10(__x); }
|
||||
__DEVICE__
|
||||
float __log2f(float __x) { return __spirv_ocl_log2(__x); }
|
||||
__DEVICE__
|
||||
float __logf(float __x) { return __spirv_ocl_log(__x); }
|
||||
__DEVICE__
|
||||
float __powf(float __x, float __y) { return __spirv_ocl_pow(__x, __y); }
|
||||
|
||||
__DEVICE__
|
||||
float __saturatef(float __x) { return __spirv_ocl_fclamp(__x, 0.0f, 1.0f); }
|
||||
|
||||
__DEVICE__
|
||||
void __sincosf(float __x, float *__sinptr, float *__cosptr) {
|
||||
*__sinptr = __spirv_ocl_sincos(__x, __cosptr);
|
||||
}
|
||||
|
||||
__DEVICE__
|
||||
float __sinf(float __x) { return __spirv_ocl_sin(__x); }
|
||||
|
||||
__DEVICE__
|
||||
float __tanf(float __x) { return __spirv_ocl_tan(__x); }
|
||||
|
||||
__DEVICE__
|
||||
int __finitef(float __x) { return !__spirv_IsInf(__x) && !__spirv_IsNan(__x); }
|
||||
__DEVICE__
|
||||
int __isinff(float __x) { return __spirv_IsInf(__x); }
|
||||
__DEVICE__
|
||||
int __isnanf(float __x) { return __spirv_IsNan(__x); }
|
||||
__DEVICE__
|
||||
int __signbitf(float __x) { return __builtin_signbitf(__x); }
|
||||
|
||||
__DEVICE__
|
||||
int __finite(double __x) { return !__spirv_IsInf(__x) && !__spirv_IsNan(__x); }
|
||||
|
||||
__DEVICE__
|
||||
int __isinf(double __x) { return __spirv_IsInf(__x); }
|
||||
|
||||
__DEVICE__
|
||||
int __isnan(double __x) { return __spirv_IsNan(__x); }
|
||||
__DEVICE__
|
||||
int __signbit(double __x) { return __builtin_signbit(__x); }
|
||||
|
||||
__DEVICE__
|
||||
double __dadd_rd(double __x, double __y) {
|
||||
double sum = __x + __y;
|
||||
double rounded = __spirv_ocl_floor(sum);
|
||||
if (rounded > sum)
|
||||
rounded -= 1.0;
|
||||
return rounded;
|
||||
}
|
||||
__DEVICE__
|
||||
double __dadd_rn(double __x, double __y) { return __spirv_ocl_rint(__x + __y); }
|
||||
__DEVICE__
|
||||
double __dadd_ru(double __x, double __y) { return __spirv_ocl_ceil(__x + __y); }
|
||||
__DEVICE__
|
||||
double __dadd_rz(double __x, double __y) {
|
||||
return __spirv_ocl_trunc(__x + __y);
|
||||
}
|
||||
__DEVICE__
|
||||
double __ddiv_rd(double __x, double __y) {
|
||||
double res = __x / __y;
|
||||
double rounded = __spirv_ocl_floor(res);
|
||||
if (rounded > res)
|
||||
rounded -= 1.0;
|
||||
return rounded;
|
||||
}
|
||||
__DEVICE__
|
||||
double __ddiv_rn(double __x, double __y) { return __spirv_ocl_rint(__x / __y); }
|
||||
__DEVICE__
|
||||
double __ddiv_ru(double __x, double __y) { return __spirv_ocl_ceil(__x / __y); }
|
||||
__DEVICE__
|
||||
double __ddiv_rz(double __x, double __y) {
|
||||
return __spirv_ocl_trunc(__x / __y);
|
||||
}
|
||||
|
||||
__DEVICE__
|
||||
double __dmul_rd(double __x, double __y) {
|
||||
double res = __x * __y;
|
||||
double rounded = __spirv_ocl_floor(res);
|
||||
if (rounded > res)
|
||||
rounded -= 1.0;
|
||||
return rounded;
|
||||
}
|
||||
__DEVICE__
|
||||
double __dmul_rn(double __x, double __y) { return __spirv_ocl_rint(__x * __y); }
|
||||
__DEVICE__
|
||||
double __dmul_ru(double __x, double __y) { return __spirv_ocl_ceil(__x * __y); }
|
||||
__DEVICE__
|
||||
double __dmul_rz(double __x, double __y) {
|
||||
return __spirv_ocl_trunc(__x * __y);
|
||||
}
|
||||
|
||||
__DEVICE__
|
||||
double __drcp_rd(double __x) { return __ddiv_rd(1.0, __x); }
|
||||
__DEVICE__
|
||||
double __drcp_rn(double __x) { return __ddiv_rn(1.0, __x); }
|
||||
__DEVICE__
|
||||
double __drcp_ru(double __x) { return __ddiv_ru(1.0, __x); }
|
||||
__DEVICE__
|
||||
double __drcp_rz(double __x) { return __ddiv_rz(1.0, __x); }
|
||||
|
||||
__DEVICE__
|
||||
double __dsqrt_rd(double __x) {
|
||||
double res = __spirv_ocl_sqrt(__x);
|
||||
double rounded = __spirv_ocl_floor(res);
|
||||
if (rounded > res)
|
||||
rounded -= 1.0;
|
||||
return rounded;
|
||||
}
|
||||
__DEVICE__
|
||||
double __dsqrt_rn(double __x) {
|
||||
return __spirv_ocl_rint(__spirv_ocl_sqrt(__x));
|
||||
}
|
||||
__DEVICE__
|
||||
double __dsqrt_ru(double __x) {
|
||||
return __spirv_ocl_ceil(__spirv_ocl_sqrt(__x));
|
||||
}
|
||||
__DEVICE__
|
||||
double __dsqrt_rz(double __x) {
|
||||
return __spirv_ocl_trunc(__spirv_ocl_sqrt(__x));
|
||||
}
|
||||
|
||||
__DEVICE__
|
||||
double __dsub_rd(double __x, double __y) {
|
||||
double res = __x - __y;
|
||||
double rounded = __spirv_ocl_floor(res);
|
||||
if (rounded > res)
|
||||
rounded -= 1.0;
|
||||
return rounded;
|
||||
}
|
||||
__DEVICE__
|
||||
double __dsub_rn(double __x, double __y) { return __spirv_ocl_rint(__x - __y); }
|
||||
__DEVICE__
|
||||
double __dsub_ru(double __x, double __y) { return __spirv_ocl_ceil(__x - __y); }
|
||||
__DEVICE__
|
||||
double __dsub_rz(double __x, double __y) {
|
||||
return __spirv_ocl_trunc(__x - __y);
|
||||
}
|
||||
|
||||
__DEVICE__
|
||||
double __fma_rd(double __x, double __y, double __z) {
|
||||
double res = __x * __y + __z;
|
||||
double rounded = __spirv_ocl_floor(res);
|
||||
if (rounded > res)
|
||||
rounded -= 1.0;
|
||||
return rounded;
|
||||
}
|
||||
__DEVICE__
|
||||
double __fma_rn(double __x, double __y, double __z) {
|
||||
return __spirv_ocl_rint(__x * __y + __z);
|
||||
}
|
||||
__DEVICE__
|
||||
double __fma_ru(double __x, double __y, double __z) {
|
||||
return __spirv_ocl_ceil(__x * __y + __z);
|
||||
}
|
||||
__DEVICE__
|
||||
double __fma_rz(double __x, double __y, double __z) {
|
||||
return __spirv_ocl_trunc(__x * __y + __z);
|
||||
}
|
||||
|
||||
__DEVICE__ int abs(int __a) { return __spirv_ocl_s_abs(__a); }
|
||||
__DEVICE__ double fabs(double __a) { return __spirv_ocl_fabs(__a); }
|
||||
__DEVICE__ double acos(double __a) { return __spirv_ocl_acos(__a); }
|
||||
__DEVICE__ float acosf(float __a) { return __spirv_ocl_acos(__a); }
|
||||
__DEVICE__ double acosh(double __a) { return __spirv_ocl_acosh(__a); }
|
||||
__DEVICE__ float acoshf(float __a) { return __spirv_ocl_acosh(__a); }
|
||||
__DEVICE__ double asin(double __a) { return __spirv_ocl_asin(__a); }
|
||||
__DEVICE__ float asinf(float __a) { return __spirv_ocl_asin(__a); }
|
||||
__DEVICE__ double asinh(double __a) { return __spirv_ocl_asinh(__a); }
|
||||
__DEVICE__ float asinhf(float __a) { return __spirv_ocl_asinh(__a); }
|
||||
__DEVICE__ double atan(double __a) { return __spirv_ocl_atan(__a); }
|
||||
__DEVICE__ double atan2(double __a, double __b) {
|
||||
return __spirv_ocl_atan2(__a, __b);
|
||||
}
|
||||
__DEVICE__ float atan2f(float __a, float __b) {
|
||||
return __spirv_ocl_atan2(__a, __b);
|
||||
}
|
||||
__DEVICE__ float atanf(float __a) { return __spirv_ocl_atan(__a); }
|
||||
__DEVICE__ double atanh(double __a) { return __spirv_ocl_atanh(__a); }
|
||||
__DEVICE__ float atanhf(float __a) { return __spirv_ocl_atanh(__a); }
|
||||
__DEVICE__ double cbrt(double __a) { return __spirv_ocl_cbrt(__a); }
|
||||
__DEVICE__ float cbrtf(float __a) { return __spirv_ocl_cbrt(__a); }
|
||||
__DEVICE__ double ceil(double __a) { return __spirv_ocl_ceil(__a); }
|
||||
__DEVICE__ float ceilf(float __a) { return __spirv_ocl_ceil(__a); }
|
||||
__DEVICE__ double copysign(double __a, double __b) {
|
||||
return __spirv_ocl_copysign(__a, __b);
|
||||
}
|
||||
__DEVICE__ float copysignf(float __a, float __b) {
|
||||
return __spirv_ocl_copysign(__a, __b);
|
||||
}
|
||||
__DEVICE__ double cos(double __a) { return __spirv_ocl_cos(__a); }
|
||||
__DEVICE__ float cosf(float __a) { return __spirv_ocl_cos(__a); }
|
||||
__DEVICE__ double cosh(double __a) { return __spirv_ocl_cosh(__a); }
|
||||
__DEVICE__ float coshf(float __a) { return __spirv_ocl_cosh(__a); }
|
||||
__DEVICE__ double cospi(double __a) { return __spirv_ocl_cospi(__a); }
|
||||
__DEVICE__ float cospif(float __a) { return __spirv_ocl_cospi(__a); }
|
||||
__DEVICE__ double erf(double __a) { return __spirv_ocl_erf(__a); }
|
||||
__DEVICE__ double erfc(double __a) { return __spirv_ocl_erfc(__a); }
|
||||
__DEVICE__ float erfcf(float __a) { return __spirv_ocl_erfc(__a); }
|
||||
__DEVICE__ double erfcx(double __a) {
|
||||
return __spirv_ocl_exp(__a * __a) * __spirv_ocl_erfc(__a);
|
||||
}
|
||||
__DEVICE__ float erfcxf(float __a) {
|
||||
return __spirv_ocl_exp(__a * __a) * __spirv_ocl_erfc(__a);
|
||||
}
|
||||
__DEVICE__ float erff(float __a) { return __spirv_ocl_erf(__a); }
|
||||
__DEVICE__ double exp(double __a) { return __spirv_ocl_exp(__a); }
|
||||
__DEVICE__ double exp10(double __a) { return __spirv_ocl_exp10(__a); }
|
||||
__DEVICE__ float exp10f(float __a) { return __spirv_ocl_exp10(__a); }
|
||||
__DEVICE__ double exp2(double __a) { return __spirv_ocl_exp2(__a); }
|
||||
__DEVICE__ float exp2f(float __a) { return __spirv_ocl_exp2(__a); }
|
||||
__DEVICE__ float expf(float __a) { return __spirv_ocl_exp(__a); }
|
||||
__DEVICE__ double expm1(double __a) { return __spirv_ocl_expm1(__a); }
|
||||
__DEVICE__ float expm1f(float __a) { return __spirv_ocl_expm1(__a); }
|
||||
__DEVICE__ float fabsf(float __a) { return __spirv_ocl_fabs(__a); }
|
||||
__DEVICE__ double fdim(double __a, double __b) {
|
||||
return __spirv_ocl_fdim(__a, __b);
|
||||
}
|
||||
__DEVICE__ float fdimf(float __a, float __b) {
|
||||
return __spirv_ocl_fdim(__a, __b);
|
||||
}
|
||||
__DEVICE__ double fdivide(double __a, double __b) { return __a / __b; }
|
||||
__DEVICE__ float fdividef(float __a, float __b) { return __a / __b; }
|
||||
__DEVICE__ double floor(double __f) { return __spirv_ocl_floor(__f); }
|
||||
__DEVICE__ float floorf(float __f) { return __spirv_ocl_floor(__f); }
|
||||
__DEVICE__ double fma(double __a, double __b, double __c) {
|
||||
return __spirv_ocl_fma(__a, __b, __c);
|
||||
}
|
||||
__DEVICE__ float fmaf(float __a, float __b, float __c) {
|
||||
return __spirv_ocl_fma(__a, __b, __c);
|
||||
}
|
||||
__DEVICE__ double fmax(double __a, double __b) {
|
||||
return __spirv_ocl_fmax(__a, __b);
|
||||
}
|
||||
__DEVICE__ float fmaxf(float __a, float __b) {
|
||||
return __spirv_ocl_fmax(__a, __b);
|
||||
}
|
||||
__DEVICE__ double fmin(double __a, double __b) {
|
||||
return __spirv_ocl_fmin(__a, __b);
|
||||
}
|
||||
__DEVICE__ float fminf(float __a, float __b) {
|
||||
return __spirv_ocl_fmin(__a, __b);
|
||||
}
|
||||
__DEVICE__ double fmod(double __a, double __b) {
|
||||
return __spirv_ocl_fmod(__a, __b);
|
||||
}
|
||||
__DEVICE__ float fmodf(float __a, float __b) {
|
||||
return __spirv_ocl_fmod(__a, __b);
|
||||
}
|
||||
__DEVICE__ double frexp(double __a, int *__b) {
|
||||
return __spirv_ocl_frexp(__a, __b);
|
||||
}
|
||||
__DEVICE__ float frexpf(float __a, int *__b) {
|
||||
return __spirv_ocl_frexp(__a, __b);
|
||||
}
|
||||
__DEVICE__ double hypot(double __a, double __b) {
|
||||
return __spirv_ocl_hypot(__a, __b);
|
||||
}
|
||||
__DEVICE__ float hypotf(float __a, float __b) {
|
||||
return __spirv_ocl_hypot(__a, __b);
|
||||
}
|
||||
__DEVICE__ int ilogb(double __a) { return __spirv_ocl_ilogb(__a); }
|
||||
__DEVICE__ int ilogbf(float __a) { return __spirv_ocl_ilogb(__a); }
|
||||
__DEVICE__ long labs(long __a) { return __spirv_ocl_s_abs(__a); };
|
||||
__DEVICE__ double ldexp(double __a, int __b) {
|
||||
return __spirv_ocl_ldexp(__a, __b);
|
||||
}
|
||||
__DEVICE__ float ldexpf(float __a, int __b) {
|
||||
return __spirv_ocl_ldexp(__a, __b);
|
||||
}
|
||||
__DEVICE__ double lgamma(double __a) { return __spirv_ocl_lgamma(__a); }
|
||||
__DEVICE__ float lgammaf(float __a) { return __spirv_ocl_lgamma(__a); }
|
||||
__DEVICE__ long long llabs(long long __a) { return __spirv_ocl_s_abs(__a); }
|
||||
__DEVICE__ long long llmax(long long __a, long long __b) {
|
||||
return __spirv_ocl_s_max(__a, __b);
|
||||
}
|
||||
__DEVICE__ long long llmin(long long __a, long long __b) {
|
||||
return __spirv_ocl_s_min(__a, __b);
|
||||
}
|
||||
__DEVICE__ long long llrint(double __a) { return __builtin_rint(__a); }
|
||||
__DEVICE__ long long llrintf(float __a) { return __builtin_rintf(__a); }
|
||||
__DEVICE__ long long llround(double __a) { return __builtin_round(__a); }
|
||||
__DEVICE__ long long llroundf(float __a) { return __builtin_roundf(__a); }
|
||||
__DEVICE__ double round(double __a) { return __spirv_ocl_round(__a); }
|
||||
__DEVICE__ float roundf(float __a) { return __spirv_ocl_round(__a); }
|
||||
__DEVICE__ double log(double __a) { return __spirv_ocl_log(__a); }
|
||||
__DEVICE__ double log10(double __a) { return __spirv_ocl_log10(__a); }
|
||||
__DEVICE__ float log10f(float __a) { return __spirv_ocl_log10(__a); }
|
||||
__DEVICE__ double log1p(double __a) { return __spirv_ocl_log1p(__a); }
|
||||
__DEVICE__ float log1pf(float __a) { return __spirv_ocl_log1p(__a); }
|
||||
__DEVICE__ double log2(double __a) { return __spirv_ocl_log2(__a); }
|
||||
__DEVICE__ float log2f(float __a) { return __spirv_ocl_log2(__a); }
|
||||
__DEVICE__ double logb(double __a) { return __spirv_ocl_logb(__a); }
|
||||
__DEVICE__ float logbf(float __a) { return __spirv_ocl_logb(__a); }
|
||||
__DEVICE__ float logf(float __a) { return __spirv_ocl_log(__a); }
|
||||
__DEVICE__ long lrint(double __a) { return __builtin_rint(__a); }
|
||||
__DEVICE__ long lrintf(float __a) { return __builtin_rintf(__a); }
|
||||
__DEVICE__ long lround(double __a) { return __builtin_round(__a); }
|
||||
__DEVICE__ long lroundf(float __a) { return __builtin_roundf(__a); }
|
||||
__DEVICE__ int max(int __a, int __b) { return __spirv_ocl_s_max(__a, __b); }
|
||||
__DEVICE__ int min(int __a, int __b) { return __spirv_ocl_s_min(__a, __b); }
|
||||
__DEVICE__ double modf(double __a, double *__b) {
|
||||
return __spirv_ocl_modf(__a, __b);
|
||||
}
|
||||
__DEVICE__ float modff(float __a, float *__b) {
|
||||
return __spirv_ocl_modf(__a, __b);
|
||||
}
|
||||
__DEVICE__ double nearbyint(double __a) { return __spirv_ocl_rint(__a); }
|
||||
__DEVICE__ float nearbyintf(float __a) { return __spirv_ocl_rint(__a); }
|
||||
__DEVICE__ double nextafter(double __a, double __b) {
|
||||
return __spirv_ocl_nextafter(__a, __b);
|
||||
}
|
||||
__DEVICE__ float nextafterf(float __a, float __b) {
|
||||
return __spirv_ocl_nextafter(__a, __b);
|
||||
}
|
||||
|
||||
__DEVICE__ double norm(int __dim, const double *__a) {
|
||||
double __r = 0;
|
||||
while (__dim--) {
|
||||
__r += __a[0] * __a[0];
|
||||
++__a;
|
||||
}
|
||||
|
||||
return __spirv_ocl_sqrt(__r);
|
||||
}
|
||||
__DEVICE__ double norm3d(double __a, double __b, double __c) {
|
||||
return __spirv_ocl_sqrt(__a * __a + __b * __b + __c * __c);
|
||||
}
|
||||
__DEVICE__ float norm3df(float __a, float __b, float __c) {
|
||||
return __spirv_ocl_sqrt(__a * __a + __b * __b + __c * __c);
|
||||
}
|
||||
__DEVICE__ double norm4d(double __a, double __b, double __c, double __d) {
|
||||
return __spirv_ocl_sqrt(__a * __a + __b * __b + __c * __c + __d * __d);
|
||||
}
|
||||
__DEVICE__ float norm4df(float __a, float __b, float __c, float __d) {
|
||||
return __spirv_ocl_sqrt(__a * __a + __b * __b + __c * __c + __d * __d);
|
||||
}
|
||||
__DEVICE__ double normcdf(double __a) {
|
||||
return 0.5 * (1.0 + __spirv_ocl_erf(__a * __spirv_ocl_rsqrt(2.0)));
|
||||
}
|
||||
__DEVICE__ float normcdff(float __a) {
|
||||
return 0.5f * (1.0f + __spirv_ocl_erf(__a * __spirv_ocl_rsqrt(2.0f)));
|
||||
}
|
||||
__DEVICE__ float normf(int __dim, const float *__a) {
|
||||
float __r = 0;
|
||||
while (__dim--) {
|
||||
__r += __a[0] * __a[0];
|
||||
++__a;
|
||||
}
|
||||
return __spirv_ocl_sqrt(__r);
|
||||
}
|
||||
__DEVICE__ double pow(double __a, double __b) {
|
||||
return __spirv_ocl_pow(__a, __b);
|
||||
}
|
||||
__DEVICE__ float powf(float __a, float __b) {
|
||||
return __spirv_ocl_pow(__a, __b);
|
||||
}
|
||||
__DEVICE__ double powi(double __a, int __b) { return pow(__a, (double)__b); }
|
||||
__DEVICE__ float powif(float __a, int __b) { return pow(__a, (float)__b); }
|
||||
__DEVICE__ double rcbrt(double __a) { return 1.0 / __spirv_ocl_cbrt(__a); }
|
||||
__DEVICE__ float rcbrtf(float __a) { return 1.0f / __spirv_ocl_cbrt(__a); }
|
||||
__DEVICE__ double remainder(double __a, double __b) {
|
||||
return __spirv_ocl_remainder(__a, __b);
|
||||
}
|
||||
__DEVICE__ float remainderf(float __a, float __b) {
|
||||
return __spirv_ocl_remainder(__a, __b);
|
||||
}
|
||||
__DEVICE__ double remquo(double __a, double __b, int *__c) {
|
||||
return __spirv_ocl_remquo(__a, __b, __c);
|
||||
}
|
||||
__DEVICE__ float remquof(float __a, float __b, int *__c) {
|
||||
return __spirv_ocl_remquo(__a, __b, __c);
|
||||
}
|
||||
__DEVICE__ double rhypot(double __a, double __b) {
|
||||
return __spirv_ocl_hypot(__a, __b);
|
||||
}
|
||||
__DEVICE__ float rhypotf(float __a, float __b) {
|
||||
return __spirv_ocl_hypot(__a, __b);
|
||||
}
|
||||
__DEVICE__ double rint(double __a) { return __spirv_ocl_rint(__a); }
|
||||
__DEVICE__ float rintf(float __a) { return __spirv_ocl_rint(__a); }
|
||||
__DEVICE__ double rnorm(int __dim, const double *__a) {
|
||||
double __r = 0;
|
||||
while (__dim--) {
|
||||
__r += __a[0] * __a[0];
|
||||
++__a;
|
||||
}
|
||||
|
||||
return __spirv_ocl_rsqrt(__r);
|
||||
}
|
||||
__DEVICE__ double rnorm3d(double __a, double __b, double __c) {
|
||||
return __spirv_ocl_rsqrt(__a * __a + __b * __b + __c * __c);
|
||||
}
|
||||
__DEVICE__ float rnorm3df(float __a, float __b, float __c) {
|
||||
return __spirv_ocl_rsqrt(__a * __a + __b * __b + __c * __c);
|
||||
}
|
||||
__DEVICE__ double rnorm4d(double __a, double __b, double __c, double __d) {
|
||||
return __spirv_ocl_rsqrt(__a * __a + __b * __b + __c * __c + __d * __d);
|
||||
}
|
||||
__DEVICE__ float rnorm4df(float __a, float __b, float __c, float __d) {
|
||||
return __spirv_ocl_rsqrt(__a * __a + __b * __b + __c * __c + __d * __d);
|
||||
}
|
||||
__DEVICE__ float rnormf(int __dim, const float *__a) {
|
||||
float __r = 0;
|
||||
while (__dim--) {
|
||||
__r += __a[0] * __a[0];
|
||||
++__a;
|
||||
}
|
||||
|
||||
return __spirv_ocl_rsqrt(__r);
|
||||
}
|
||||
__DEVICE__ double rsqrt(double __a) { return __spirv_ocl_rsqrt(__a); }
|
||||
__DEVICE__ float rsqrtf(float __a) { return __spirv_ocl_rsqrt(__a); }
|
||||
__DEVICE__ double scalbn(double __a, int __b) {
|
||||
return __spirv_ocl_ldexp(__a, __b);
|
||||
}
|
||||
__DEVICE__ float scalbnf(float __a, int __b) {
|
||||
return __spirv_ocl_ldexp(__a, __b);
|
||||
}
|
||||
__DEVICE__ double scalbln(double __a, long __b) {
|
||||
if (__b > INT_MAX)
|
||||
return __a > 0 ? HUGE_VAL : -HUGE_VAL;
|
||||
if (__b < INT_MIN)
|
||||
return __a > 0 ? 0.0 : -0.0;
|
||||
return scalbn(__a, (int)__b);
|
||||
}
|
||||
__DEVICE__ float scalblnf(float __a, long __b) {
|
||||
if (__b > INT_MAX)
|
||||
return __a > 0 ? HUGE_VALF : -HUGE_VALF;
|
||||
if (__b < INT_MIN)
|
||||
return __a > 0 ? 0.f : -0.f;
|
||||
return scalbnf(__a, (int)__b);
|
||||
}
|
||||
__DEVICE__ double sin(double __a) { return __spirv_ocl_sin(__a); }
|
||||
__DEVICE__ void sincos(double __a, double *__s, double *__c) {
|
||||
*__s = __spirv_ocl_sincos(__a, __c);
|
||||
}
|
||||
__DEVICE__ void sincosf(float __a, float *__s, float *__c) {
|
||||
*__s = __spirv_ocl_sincos(__a, __c);
|
||||
}
|
||||
__DEVICE__ void sincospi(double __a, double *__s, double *__c) {
|
||||
*__s = __spirv_ocl_sinpi(__a);
|
||||
*__c = __spirv_ocl_cospi(__a);
|
||||
}
|
||||
__DEVICE__ void sincospif(float __a, float *__s, float *__c) {
|
||||
*__s = __spirv_ocl_sinpi(__a);
|
||||
*__c = __spirv_ocl_cospi(__a);
|
||||
}
|
||||
__DEVICE__ float sinf(float __a) { return __spirv_ocl_sin(__a); }
|
||||
__DEVICE__ double sinh(double __a) { return __spirv_ocl_sinh(__a); }
|
||||
__DEVICE__ float sinhf(float __a) { return __spirv_ocl_sinh(__a); }
|
||||
__DEVICE__ double sinpi(double __a) { return __spirv_ocl_sinpi(__a); }
|
||||
__DEVICE__ float sinpif(float __a) { return __spirv_ocl_sinpi(__a); }
|
||||
__DEVICE__ double sqrt(double __a) { return __spirv_ocl_sqrt(__a); }
|
||||
__DEVICE__ float sqrtf(float __a) { return __spirv_ocl_sqrt(__a); }
|
||||
__DEVICE__ double tan(double __a) { return __spirv_ocl_tan(__a); }
|
||||
__DEVICE__ float tanf(float __a) { return __spirv_ocl_tan(__a); }
|
||||
__DEVICE__ double tanh(double __a) { return __spirv_ocl_tanh(__a); }
|
||||
__DEVICE__ float tanhf(float __a) { return __spirv_ocl_tanh(__a); }
|
||||
__DEVICE__ double tgamma(double __a) { return __spirv_ocl_tgamma(__a); }
|
||||
__DEVICE__ float tgammaf(float __a) { return __spirv_ocl_tgamma(__a); }
|
||||
__DEVICE__ double trunc(double __a) { return __spirv_ocl_trunc(__a); }
|
||||
__DEVICE__ float truncf(float __a) { return __spirv_ocl_trunc(__a); }
|
||||
__DEVICE__ unsigned long long ullmax(unsigned long long __a,
|
||||
unsigned long long __b) {
|
||||
return __spirv_ocl_u_max(__a, __b);
|
||||
}
|
||||
__DEVICE__ unsigned long long ullmin(unsigned long long __a,
|
||||
unsigned long long __b) {
|
||||
return __spirv_ocl_u_min(__a, __b);
|
||||
}
|
||||
__DEVICE__ unsigned int umax(unsigned int __a, unsigned int __b) {
|
||||
return __spirv_ocl_u_max(__a, __b);
|
||||
}
|
||||
__DEVICE__ unsigned int umin(unsigned int __a, unsigned int __b) {
|
||||
return __spirv_ocl_u_min(__a, __b);
|
||||
}
|
||||
|
||||
#if !defined(__cplusplus) && __STDC_VERSION__ >= 201112L
|
||||
#define isfinite(__x) _Generic((__x), float: __finitef, double: __finite)(__x)
|
||||
#define isinf(__x) _Generic((__x), float: __isinff, double: __isinf)(__x)
|
||||
#define isnan(__x) _Generic((__x), float: __isnanf, double: __isnan)(__x)
|
||||
#define signbit(__x) _Generic((__x), float: __signbitf, double: __signbit)(__x)
|
||||
#endif // !defined(__cplusplus) && __STDC_VERSION__ >= 201112L
|
||||
|
||||
__DEVICE__
|
||||
unsigned long __make_mantissa_base8(const char *__tagp) {
|
||||
unsigned long __r = 0;
|
||||
while (*__tagp != '\0') {
|
||||
char __tmp = *__tagp;
|
||||
|
||||
if (__tmp >= '0' && __tmp <= '7')
|
||||
__r = (__r * 8u) + __tmp - '0';
|
||||
else
|
||||
return 0;
|
||||
|
||||
++__tagp;
|
||||
}
|
||||
|
||||
return __r;
|
||||
}
|
||||
|
||||
__DEVICE__
|
||||
unsigned long __make_mantissa_base10(const char *__tagp) {
|
||||
unsigned long __r = 0;
|
||||
while (*__tagp != '\0') {
|
||||
char __tmp = *__tagp;
|
||||
|
||||
if (__tmp >= '0' && __tmp <= '9')
|
||||
__r = (__r * 10u) + __tmp - '0';
|
||||
else
|
||||
return 0;
|
||||
|
||||
++__tagp;
|
||||
}
|
||||
|
||||
return __r;
|
||||
}
|
||||
|
||||
__DEVICE__
|
||||
unsigned long __make_mantissa_base16(const char *__tagp) {
|
||||
unsigned long __r = 0;
|
||||
while (*__tagp != '\0') {
|
||||
char __tmp = *__tagp;
|
||||
|
||||
if (__tmp >= '0' && __tmp <= '9')
|
||||
__r = (__r * 16u) + __tmp - '0';
|
||||
else if (__tmp >= 'a' && __tmp <= 'f')
|
||||
__r = (__r * 16u) + __tmp - 'a' + 10;
|
||||
else if (__tmp >= 'A' && __tmp <= 'F')
|
||||
__r = (__r * 16u) + __tmp - 'A' + 10;
|
||||
else
|
||||
return 0;
|
||||
|
||||
++__tagp;
|
||||
}
|
||||
|
||||
return __r;
|
||||
}
|
||||
|
||||
__DEVICE__
|
||||
unsigned long __make_mantissa(const char *__tagp) {
|
||||
if (!__tagp)
|
||||
return 0;
|
||||
if (*__tagp == '0') {
|
||||
++__tagp;
|
||||
|
||||
if (*__tagp == 'x' || *__tagp == 'X')
|
||||
return __make_mantissa_base16(++__tagp);
|
||||
else
|
||||
return __make_mantissa_base8(__tagp);
|
||||
}
|
||||
|
||||
return __make_mantissa_base10(__tagp);
|
||||
}
|
||||
|
||||
float nanf(const char *__tagp) {
|
||||
return __spirv_ocl_nan((unsigned int)__make_mantissa(__tagp));
|
||||
}
|
||||
double nan(const char *__tagp) {
|
||||
return __spirv_ocl_nan(__make_mantissa(__tagp));
|
||||
}
|
||||
|
||||
#pragma pop_macro("__DEVICE__")
|
||||
#endif // __CLANG_GPU_DISABLE_MATH_WRAPPERS
|
||||
#endif // __CLANG_SPIRV_MATH_H__
|
||||
Vendored
+79
-9
@@ -2117,7 +2117,7 @@ vec_cmpne(vector unsigned long long __a, vector unsigned long long __b) {
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef __VSX__
|
||||
#ifdef __POWER8_VECTOR__
|
||||
static __inline__ vector bool long long __ATTRS_o_ai
|
||||
vec_cmpne(vector double __a, vector double __b) {
|
||||
return (vector bool long long)
|
||||
@@ -6404,6 +6404,18 @@ vec_mulh(vector unsigned long long __a, vector unsigned long long __b) {
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef __FUTURE_VECTOR__
|
||||
static __inline__ vector signed short
|
||||
__ATTRS_o_ai vec_mulh(vector signed short __a, vector signed short __b) {
|
||||
return __builtin_altivec_vmulhsh(__a, __b);
|
||||
}
|
||||
|
||||
static __inline__ vector unsigned short __ATTRS_o_ai
|
||||
vec_mulh(vector unsigned short __a, vector unsigned short __b) {
|
||||
return __builtin_altivec_vmulhuh(__a, __b);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* vec_mulo */
|
||||
|
||||
static __inline__ vector short __ATTRS_o_ai vec_mulo(vector signed char __a,
|
||||
@@ -17809,6 +17821,7 @@ vec_xl(ptrdiff_t __offset, const unsigned __int128 *__ptr) {
|
||||
/* vec_xl_be */
|
||||
|
||||
#ifdef __LITTLE_ENDIAN__
|
||||
#ifdef __VSX__
|
||||
static __inline__ vector signed char __ATTRS_o_ai
|
||||
vec_xl_be(ptrdiff_t __offset, const signed char *__ptr) {
|
||||
vector signed char __vec = (vector signed char)__builtin_vsx_lxvd2x_be(__offset, __ptr);
|
||||
@@ -17850,7 +17863,6 @@ vec_xl_be(signed long long __offset, const float *__ptr) {
|
||||
return (vector float)__builtin_vsx_lxvw4x_be(__offset, __ptr);
|
||||
}
|
||||
|
||||
#ifdef __VSX__
|
||||
static __inline__ vector signed long long __ATTRS_o_ai
|
||||
vec_xl_be(signed long long __offset, const signed long long *__ptr) {
|
||||
return (vector signed long long)__builtin_vsx_lxvd2x_be(__offset, __ptr);
|
||||
@@ -17865,7 +17877,6 @@ static __inline__ vector double __ATTRS_o_ai
|
||||
vec_xl_be(signed long long __offset, const double *__ptr) {
|
||||
return (vector double)__builtin_vsx_lxvd2x_be(__offset, __ptr);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__POWER8_VECTOR__) && defined(__powerpc64__) && \
|
||||
defined(__SIZEOF_INT128__)
|
||||
@@ -17879,8 +17890,9 @@ vec_xl_be(signed long long __offset, const unsigned __int128 *__ptr) {
|
||||
return vec_xl(__offset, __ptr);
|
||||
}
|
||||
#endif
|
||||
#else
|
||||
#define vec_xl_be vec_xl
|
||||
#endif // __VSX__
|
||||
#else // ! __LITTLE_ENDIAN__
|
||||
#define vec_xl_be vec_xl
|
||||
#endif
|
||||
|
||||
#if defined(__POWER10_VECTOR__) && defined(__VSX__) && \
|
||||
@@ -18130,6 +18142,7 @@ vec_xst_trunc(vector unsigned __int128 __vec, ptrdiff_t __offset,
|
||||
/* vec_xst_be */
|
||||
|
||||
#ifdef __LITTLE_ENDIAN__
|
||||
#ifdef __VSX__
|
||||
static __inline__ void __ATTRS_o_ai vec_xst_be(vector signed char __vec,
|
||||
signed long long __offset,
|
||||
signed char *__ptr) {
|
||||
@@ -18186,7 +18199,6 @@ static __inline__ void __ATTRS_o_ai vec_xst_be(vector float __vec,
|
||||
__builtin_vsx_stxvw4x_be((vector int)__vec, __offset, __ptr);
|
||||
}
|
||||
|
||||
#ifdef __VSX__
|
||||
static __inline__ void __ATTRS_o_ai vec_xst_be(vector signed long long __vec,
|
||||
signed long long __offset,
|
||||
signed long long *__ptr) {
|
||||
@@ -18204,7 +18216,6 @@ static __inline__ void __ATTRS_o_ai vec_xst_be(vector double __vec,
|
||||
double *__ptr) {
|
||||
__builtin_vsx_stxvd2x_be((vector double)__vec, __offset, __ptr);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__POWER8_VECTOR__) && defined(__powerpc64__) && \
|
||||
defined(__SIZEOF_INT128__)
|
||||
@@ -18220,8 +18231,9 @@ static __inline__ void __ATTRS_o_ai vec_xst_be(vector unsigned __int128 __vec,
|
||||
vec_xst(__vec, __offset, __ptr);
|
||||
}
|
||||
#endif
|
||||
#else
|
||||
#define vec_xst_be vec_xst
|
||||
#endif // VSX
|
||||
#else // ! __LITTLE_ENDIAN__
|
||||
#define vec_xst_be vec_xst
|
||||
#endif
|
||||
|
||||
#ifdef __POWER9_VECTOR__
|
||||
@@ -19314,6 +19326,64 @@ vec_sra(vector signed __int128 __a, vector unsigned __int128 __b) {
|
||||
#endif /* __SIZEOF_INT128__ */
|
||||
#endif /* __POWER10_VECTOR__ */
|
||||
|
||||
#ifdef __FUTURE_VECTOR__
|
||||
|
||||
/* vec_uncompress* - Deeply Compressed Weights builtins */
|
||||
|
||||
static __inline__ vector unsigned char __ATTRS_o_ai
|
||||
vec_uncompresshn(vector unsigned char __a, vector unsigned char __b) {
|
||||
return __builtin_altivec_vucmprhn(__a, __b);
|
||||
}
|
||||
|
||||
static __inline__ vector unsigned char __ATTRS_o_ai
|
||||
vec_uncompressln(vector unsigned char __a, vector unsigned char __b) {
|
||||
return __builtin_altivec_vucmprln(__a, __b);
|
||||
}
|
||||
|
||||
static __inline__ vector unsigned char __ATTRS_o_ai
|
||||
vec_uncompresshb(vector unsigned char __a, vector unsigned char __b) {
|
||||
return __builtin_altivec_vucmprhb(__a, __b);
|
||||
}
|
||||
|
||||
static __inline__ vector unsigned char __ATTRS_o_ai
|
||||
vec_uncompresslb(vector unsigned char __a, vector unsigned char __b) {
|
||||
return __builtin_altivec_vucmprlb(__a, __b);
|
||||
}
|
||||
|
||||
static __inline__ vector unsigned char __ATTRS_o_ai
|
||||
vec_uncompresshh(vector unsigned char __a, vector unsigned char __b) {
|
||||
return __builtin_altivec_vucmprhh(__a, __b);
|
||||
}
|
||||
|
||||
static __inline__ vector unsigned char __ATTRS_o_ai
|
||||
vec_uncompresslh(vector unsigned char __a, vector unsigned char __b) {
|
||||
return __builtin_altivec_vucmprlh(__a, __b);
|
||||
}
|
||||
|
||||
static __inline__ vector unsigned char __ATTRS_o_ai
|
||||
vec_unpack_hsn_to_byte(vector unsigned char __a) {
|
||||
return __builtin_altivec_vupkhsntob(__a);
|
||||
}
|
||||
|
||||
static __inline__ vector unsigned char __ATTRS_o_ai
|
||||
vec_unpack_lsn_to_byte(vector unsigned char __a) {
|
||||
return __builtin_altivec_vupklsntob(__a);
|
||||
}
|
||||
|
||||
#define vec_unpack_int4_to_bf16(__a, __imm) \
|
||||
__builtin_altivec_vupkint4tobf16((__a), (__imm))
|
||||
|
||||
#define vec_unpack_int8_to_bf16(__a, __imm) \
|
||||
__builtin_altivec_vupkint8tobf16((__a), (__imm))
|
||||
|
||||
#define vec_unpack_int4_to_fp32(__a, __imm) \
|
||||
__builtin_altivec_vupkint4tofp32((__a), (__imm))
|
||||
|
||||
#define vec_unpack_int8_to_fp32(__a, __imm) \
|
||||
__builtin_altivec_vupkint8tofp32((__a), (__imm))
|
||||
|
||||
#endif /* __FUTURE_VECTOR__ */
|
||||
|
||||
#ifdef __POWER8_VECTOR__
|
||||
#define __bcdadd(__a, __b, __ps) __builtin_ppc_bcdadd((__a), (__b), (__ps))
|
||||
#define __bcdsub(__a, __b, __ps) __builtin_ppc_bcdsub((__a), (__b), (__ps))
|
||||
|
||||
Vendored
+83
@@ -0,0 +1,83 @@
|
||||
//===-- amdhsa_abi.h - AMDHSA ABI definition utilities --------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __AMDHSA_ABI_H
|
||||
#define __AMDHSA_ABI_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
typedef struct __attribute__((aligned(8))) amdhsa_implicit_kernarg_v5 {
|
||||
uint32_t block_count[3];
|
||||
uint16_t group_size[3];
|
||||
uint16_t remainder[3];
|
||||
char reserved0[16];
|
||||
uint64_t global_offset[3];
|
||||
uint16_t grid_dims;
|
||||
char reserved1[6];
|
||||
__attribute__((opencl_global)) void *printf_buffer;
|
||||
__attribute__((opencl_global)) void *hostcall_buffer;
|
||||
__attribute__((opencl_global)) void *multigrid_sync_arg;
|
||||
__attribute__((opencl_global)) void *heap_v1;
|
||||
__attribute__((opencl_global)) void *default_queue;
|
||||
__attribute__((opencl_global)) void *completion_action;
|
||||
char reserved2[72];
|
||||
uint32_t private_base; // Unused on gfx9+
|
||||
uint32_t shared_base; // Unused on gfx9+
|
||||
__attribute__((opencl_global)) void *queue_ptr;
|
||||
char reserved3[48];
|
||||
} amdhsa_implicit_kernarg_v5;
|
||||
|
||||
_Static_assert(sizeof(amdhsa_implicit_kernarg_v5) == 256, "wrong struct size");
|
||||
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, block_count[0]) == 0,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, block_count[1]) == 4,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, block_count[2]) == 8,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, group_size[0]) == 12,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, group_size[1]) == 14,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, group_size[2]) == 16,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, remainder[0]) == 18,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, remainder[1]) == 20,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, remainder[2]) == 22,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, global_offset[0]) == 40,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, global_offset[1]) == 48,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, global_offset[2]) == 56,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, grid_dims) == 64,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, printf_buffer) == 72,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, hostcall_buffer) == 80,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, multigrid_sync_arg) == 88,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, heap_v1) == 96,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, default_queue) == 104,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, completion_action) == 112,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, private_base) == 192,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, shared_base) == 196,
|
||||
"wrong offset");
|
||||
_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, queue_ptr) == 200,
|
||||
"wrong offset");
|
||||
|
||||
#endif // __AMDHSA_ABI_H
|
||||
Vendored
+89
@@ -124,6 +124,95 @@ static inline int64_t amo_ldat_sswap(int64_t *ptr, int64_t val) {
|
||||
return __builtin_amo_ldat_s(ptr, val, _AMO_LD_SWAP);
|
||||
}
|
||||
|
||||
/* AMO Store Operation Codes (FC values) */
|
||||
enum _AMO_ST {
|
||||
_AMO_ST_ADD = 0x00, /* Store Add */
|
||||
_AMO_ST_XOR = 0x01, /* Store Xor */
|
||||
_AMO_ST_IOR = 0x02, /* Store Ior */
|
||||
_AMO_ST_AND = 0x03, /* Store And */
|
||||
_AMO_ST_UMAX = 0x04, /* Store Unsigned Maximum */
|
||||
_AMO_ST_SMAX = 0x05, /* Store Signed Maximum */
|
||||
_AMO_ST_UMIN = 0x06, /* Store Unsigned Minimum */
|
||||
_AMO_ST_SMIN = 0x07, /* Store Signed Minimum */
|
||||
_AMO_ST_TWIN = 0x18 /* Store Twin */
|
||||
};
|
||||
|
||||
/* 32-bit unsigned AMO store operations */
|
||||
static inline void amo_stwat_add(uint32_t *ptr, uint32_t val) {
|
||||
__builtin_amo_stwat(ptr, val, _AMO_ST_ADD);
|
||||
}
|
||||
|
||||
static inline void amo_stwat_xor(uint32_t *ptr, uint32_t val) {
|
||||
__builtin_amo_stwat(ptr, val, _AMO_ST_XOR);
|
||||
}
|
||||
|
||||
static inline void amo_stwat_ior(uint32_t *ptr, uint32_t val) {
|
||||
__builtin_amo_stwat(ptr, val, _AMO_ST_IOR);
|
||||
}
|
||||
|
||||
static inline void amo_stwat_and(uint32_t *ptr, uint32_t val) {
|
||||
__builtin_amo_stwat(ptr, val, _AMO_ST_AND);
|
||||
}
|
||||
|
||||
static inline void amo_stwat_umax(uint32_t *ptr, uint32_t val) {
|
||||
__builtin_amo_stwat(ptr, val, _AMO_ST_UMAX);
|
||||
}
|
||||
|
||||
static inline void amo_stwat_umin(uint32_t *ptr, uint32_t val) {
|
||||
__builtin_amo_stwat(ptr, val, _AMO_ST_UMIN);
|
||||
}
|
||||
|
||||
/* 32-bit signed AMO store operations */
|
||||
static inline void amo_stwat_sadd(int32_t *ptr, int32_t val) {
|
||||
__builtin_amo_stwat_s(ptr, val, _AMO_ST_ADD);
|
||||
}
|
||||
|
||||
static inline void amo_stwat_smax(int32_t *ptr, int32_t val) {
|
||||
__builtin_amo_stwat_s(ptr, val, _AMO_ST_SMAX);
|
||||
}
|
||||
|
||||
static inline void amo_stwat_smin(int32_t *ptr, int32_t val) {
|
||||
__builtin_amo_stwat_s(ptr, val, _AMO_ST_SMIN);
|
||||
}
|
||||
|
||||
/* 64-bit unsigned AMO store operations */
|
||||
static inline void amo_stdat_add(uint64_t *ptr, uint64_t val) {
|
||||
__builtin_amo_stdat(ptr, val, _AMO_ST_ADD);
|
||||
}
|
||||
|
||||
static inline void amo_stdat_xor(uint64_t *ptr, uint64_t val) {
|
||||
__builtin_amo_stdat(ptr, val, _AMO_ST_XOR);
|
||||
}
|
||||
|
||||
static inline void amo_stdat_ior(uint64_t *ptr, uint64_t val) {
|
||||
__builtin_amo_stdat(ptr, val, _AMO_ST_IOR);
|
||||
}
|
||||
|
||||
static inline void amo_stdat_and(uint64_t *ptr, uint64_t val) {
|
||||
__builtin_amo_stdat(ptr, val, _AMO_ST_AND);
|
||||
}
|
||||
|
||||
static inline void amo_stdat_umax(uint64_t *ptr, uint64_t val) {
|
||||
__builtin_amo_stdat(ptr, val, _AMO_ST_UMAX);
|
||||
}
|
||||
|
||||
static inline void amo_stdat_umin(uint64_t *ptr, uint64_t val) {
|
||||
__builtin_amo_stdat(ptr, val, _AMO_ST_UMIN);
|
||||
}
|
||||
|
||||
/* 64-bit signed AMO store operations */
|
||||
static inline void amo_stdat_sadd(int64_t *ptr, int64_t val) {
|
||||
__builtin_amo_stdat_s(ptr, val, _AMO_ST_ADD);
|
||||
}
|
||||
|
||||
static inline void amo_stdat_smax(int64_t *ptr, int64_t val) {
|
||||
__builtin_amo_stdat_s(ptr, val, _AMO_ST_SMAX);
|
||||
}
|
||||
|
||||
static inline void amo_stdat_smin(int64_t *ptr, int64_t val) {
|
||||
__builtin_amo_stdat_s(ptr, val, _AMO_ST_SMIN);
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
Vendored
-108
@@ -1,108 +0,0 @@
|
||||
/*===------------- amxtf32intrin.h - AMX_TF32 intrinsics -*- C++ -*---------===
|
||||
*
|
||||
* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
* See https://llvm.org/LICENSE.txt for license information.
|
||||
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
*
|
||||
*===------------------------------------------------------------------------===
|
||||
*/
|
||||
|
||||
#ifndef __IMMINTRIN_H
|
||||
#error "Never use <amxtf32intrin.h> directly; include <immintrin.h> instead."
|
||||
#endif // __IMMINTRIN_H
|
||||
|
||||
#ifndef __AMX_TF32INTRIN_H
|
||||
#define __AMX_TF32INTRIN_H
|
||||
#ifdef __x86_64__
|
||||
|
||||
#define __DEFAULT_FN_ATTRS_TF32 \
|
||||
__attribute__((__always_inline__, __nodebug__, __target__("amx-tf32")))
|
||||
|
||||
/// Do Matrix Multiplication of \a a and \a b, and then do Matrix Plus
|
||||
/// with \a srcdst.
|
||||
/// All the calculation is base on float32 but with the lower 13-bit set to 0.
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// \code
|
||||
/// void _tile_mmultf32ps(constexpr int srcdst, constexpr int a, \
|
||||
/// constexpr int b);
|
||||
/// \endcode
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> TMMULTF32PS </c> instruction.
|
||||
///
|
||||
/// \param srcdst
|
||||
/// The destination tile. Max size is 1024 Bytes.
|
||||
/// \param a
|
||||
/// The 1st source tile. Max size is 1024 Bytes.
|
||||
/// \param b
|
||||
/// The 2nd source tile. Max size is 1024 Bytes.
|
||||
///
|
||||
/// \code{.operation}
|
||||
/// DEFINE zero_lower_mantissa_bits_fp32(x[31:0]) {
|
||||
/// dword[12:0] := 0
|
||||
/// dword[31:13] := x[31:13]
|
||||
/// return dword
|
||||
/// }
|
||||
///
|
||||
/// DEFINE silence_snan_fp32(x[31:0]) {
|
||||
/// IF (x.exponent == 255 and x.fraction != 0 and x.fraction[22] == 0)
|
||||
/// x.fraction[22] := 1
|
||||
/// return x
|
||||
/// }
|
||||
///
|
||||
/// elements_a := a.colsb / 4
|
||||
/// elements_dest := srcdst.colsb / 4
|
||||
///
|
||||
/// FOR m = 0 TO (srcdst.rows-1)
|
||||
/// tmp[511:0] := 0
|
||||
/// FOR k = 0 TO (elements_a-1)
|
||||
/// FOR n = 0 TO (elements_dest-1)
|
||||
/// af := silence_snan_fp32(a.row[m].fp32[k])
|
||||
/// bf := silence_snan_fp32(b.row[k].fp32[n])
|
||||
/// tmp.fp32[n] += zero_lower_mantissa_bits_fp32(af)
|
||||
/// * zero_lower_mantissa_bits_fp32(bf)
|
||||
/// ENDFOR
|
||||
/// ENDFOR
|
||||
///
|
||||
/// FOR n = 0 TO (elements_dest-1)
|
||||
/// tmp.fp32[n] += srcdst.row[m].fp32[n]
|
||||
/// ENDFOR
|
||||
/// write_row_and_zero(srcdst, m, tmp, srcdst.colsb)
|
||||
///
|
||||
/// ENDFOR
|
||||
///
|
||||
/// zero_upper_rows(srcdst, srcdst.rows)
|
||||
/// zero_tileconfig_start()
|
||||
/// \endcode
|
||||
#define _tile_mmultf32ps(srcdst, a, b) \
|
||||
__builtin_ia32_tmmultf32ps((srcdst), (a), (b))
|
||||
|
||||
static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TF32
|
||||
_tile_mmultf32ps_internal(unsigned short m, unsigned short n, unsigned short k,
|
||||
_tile1024i dst, _tile1024i src1, _tile1024i src2) {
|
||||
return __builtin_ia32_tmmultf32ps_internal(m, n, k, dst, src1, src2);
|
||||
}
|
||||
|
||||
/// Do Matrix Multiplication of src0 and src1, and then do Matrix Plus with dst.
|
||||
/// All the calculation is base on float32 but with the lower 13-bit set to 0.
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> TMMULTF32PS </c> instruction.
|
||||
///
|
||||
/// \param dst
|
||||
/// The destination tile. Max size is 1024 Bytes.
|
||||
/// \param src0
|
||||
/// The 1st source tile. Max size is 1024 Bytes.
|
||||
/// \param src1
|
||||
/// The 2nd source tile. Max size is 1024 Bytes.
|
||||
__DEFAULT_FN_ATTRS_TF32
|
||||
static void __tile_mmultf32ps(__tile1024i *dst, __tile1024i src0,
|
||||
__tile1024i src1) {
|
||||
dst->tile = _tile_mmultf32ps_internal(src0.row, src1.col, src0.col, dst->tile,
|
||||
src0.tile, src1.tile);
|
||||
}
|
||||
|
||||
#endif // __x86_64__
|
||||
#endif // __AMX_TF32INTRIN_H
|
||||
Vendored
+5
@@ -13,4 +13,9 @@
|
||||
|
||||
#pragma clang riscv intrinsic andes_vector
|
||||
|
||||
#define __riscv_intrinsic_xandesvbfhcvt 1
|
||||
#define __riscv_intrinsic_xandesvdot 1
|
||||
#define __riscv_intrinsic_xandesvpackfph 1
|
||||
#define __riscv_intrinsic_xandesvsintload 1
|
||||
|
||||
#endif //_ANDES_VECTOR_H_
|
||||
Vendored
+21
@@ -0,0 +1,21 @@
|
||||
/*===---- arm64_neon.h - ARM64 NEON intrinsics -----------------------------===
|
||||
*
|
||||
* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
* See https://llvm.org/LICENSE.txt for license information.
|
||||
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
*
|
||||
*===-----------------------------------------------------------------------===
|
||||
*/
|
||||
|
||||
/* Only include this if we're compiling for the windows platform. */
|
||||
#ifndef _MSC_VER
|
||||
#include_next <arm64_neon.h>
|
||||
#else
|
||||
|
||||
#ifndef __ARM64_NEON_H
|
||||
#define __ARM64_NEON_H
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#endif /* __ARM64_NEON_H */
|
||||
#endif /* _MSC_VER */
|
||||
Vendored
+10
@@ -15,6 +15,16 @@
|
||||
#ifndef __ARM64INTR_H
|
||||
#define __ARM64INTR_H
|
||||
|
||||
/* Encode an AArch64 system register for use with
|
||||
_ReadStatusReg/_WriteStatusReg. op0 must be 2 or 3; only the low bit is
|
||||
stored. */
|
||||
#define ARM64_SYSREG(op0, op1, CRn, CRm, op2) \
|
||||
((((op0) & 0x1) << 14) | (((op1) & 0x7) << 11) | (((CRn) & 0xF) << 7) | \
|
||||
(((CRm) & 0xF) << 3) | ((op2) & 0x7))
|
||||
|
||||
#define ARM64_FPCR ARM64_SYSREG(3, 3, 4, 4, 0)
|
||||
#define ARM64_FPSR ARM64_SYSREG(3, 3, 4, 4, 1)
|
||||
|
||||
typedef enum
|
||||
{
|
||||
_ARM64_BARRIER_SY = 0xF,
|
||||
|
||||
Vendored
+1
@@ -115,6 +115,7 @@ __swp(uint32_t __x, volatile uint32_t *__p) {
|
||||
#else
|
||||
#define __plix(cache_level, retention_policy, addr) \
|
||||
__builtin_arm_prefetch(addr, 0, cache_level, retention_policy, 0)
|
||||
#define __pldir(addr) __builtin_arm_prefetch_ir(addr)
|
||||
#endif
|
||||
|
||||
/* 7.7 NOP */
|
||||
|
||||
Vendored
+3465
-3297
File diff suppressed because it is too large.
Load diff
Vendored
+12
@@ -3732,6 +3732,18 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svzero_za64_vg4x2)))
|
||||
void svzero_za64_vg4x2(uint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svzero_za64_vg4x4)))
|
||||
void svzero_za64_vg4x4(uint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti6_zt_u8)))
|
||||
svuint8_t svluti6_zt_u8(uint64_t, svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti6_zt_s8)))
|
||||
svint8_t svluti6_zt_s8(uint64_t, svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti6_zt_mf8)))
|
||||
svmfloat8_t svluti6_zt_mf8(uint64_t, svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti6_zt_u8_x4)))
|
||||
svuint8x4_t svluti6_zt_u8_x4(uint64_t, svuint8x3_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti6_zt_s8_x4)))
|
||||
svint8x4_t svluti6_zt_s8_x4(uint64_t, svuint8x3_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti6_zt_mf8_x4)))
|
||||
svmfloat8x4_t svluti6_zt_mf8_x4(uint64_t, svuint8x3_t);
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
Vendored
+420
@@ -4697,6 +4697,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f32_x4)))
|
||||
svfloat32x4_t svmul(svfloat32x4_t, svfloat32x4_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f16_x4)))
|
||||
svfloat16x4_t svmul(svfloat16x4_t, svfloat16x4_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_u16_x4_u16_x2_u8_x2)))
|
||||
svuint16x4_t svluti6_lane_u16_x4_u16_x2_u8_x2(svuint16x2_t, svuint8x2_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_bf16_x4_bf16_x2_u8_x2)))
|
||||
svbfloat16x4_t svluti6_lane_bf16_x4_bf16_x2_u8_x2(svbfloat16x2_t, svuint8x2_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_f16_x4_f16_x2_u8_x2)))
|
||||
svfloat16x4_t svluti6_lane_f16_x4_f16_x2_u8_x2(svfloat16x2_t, svuint8x2_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_s16_x4_s16_x2_u8_x2)))
|
||||
svint16x4_t svluti6_lane_s16_x4_s16_x2_u8_x2(svint16x2_t, svuint8x2_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_u16_x4_u16_x2_u8_x3)))
|
||||
svuint16x4_t svluti6_lane_u16_x4_u16_x2_u8_x3(svuint16x2_t, svuint8x3_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_bf16_x4_bf16_x2_u8_x3)))
|
||||
svbfloat16x4_t svluti6_lane_bf16_x4_bf16_x2_u8_x3(svbfloat16x2_t, svuint8x3_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_f16_x4_f16_x2_u8_x3)))
|
||||
svfloat16x4_t svluti6_lane_f16_x4_f16_x2_u8_x3(svfloat16x2_t, svuint8x3_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_s16_x4_s16_x2_u8_x3)))
|
||||
svint16x4_t svluti6_lane_s16_x4_s16_x2_u8_x3(svint16x2_t, svuint8x3_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_u16_x4_u16_x2_u8_x2)))
|
||||
svuint16x4_t svluti6_lane_u16_x4(svuint16x2_t, svuint8x2_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_bf16_x4_bf16_x2_u8_x2)))
|
||||
svbfloat16x4_t svluti6_lane_bf16_x4(svbfloat16x2_t, svuint8x2_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_f16_x4_f16_x2_u8_x2)))
|
||||
svfloat16x4_t svluti6_lane_f16_x4(svfloat16x2_t, svuint8x2_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_s16_x4_s16_x2_u8_x2)))
|
||||
svint16x4_t svluti6_lane_s16_x4(svint16x2_t, svuint8x2_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_u16_x4_u16_x2_u8_x3)))
|
||||
svuint16x4_t svluti6_lane_u16_x4(svuint16x2_t, svuint8x3_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_bf16_x4_bf16_x2_u8_x3)))
|
||||
svbfloat16x4_t svluti6_lane_bf16_x4(svbfloat16x2_t, svuint8x3_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_f16_x4_f16_x2_u8_x3)))
|
||||
svfloat16x4_t svluti6_lane_f16_x4(svfloat16x2_t, svuint8x3_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_s16_x4_s16_x2_u8_x3)))
|
||||
svint16x4_t svluti6_lane_s16_x4(svint16x2_t, svuint8x3_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f64)))
|
||||
float64_t svadda_f64(svbool_t, float64_t, svfloat64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f32)))
|
||||
@@ -7223,6 +7255,12 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcreate2_b)))
|
||||
svboolx2_t svcreate2_b(svbool_t, svbool_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcreate4_b)))
|
||||
svboolx4_t svcreate4_b(svbool_t, svbool_t, svbool_t, svbool_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f32_f16)))
|
||||
svfloat32_t svdot_n_f32_f16(svfloat32_t, svfloat16_t, float16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_s32_s16)))
|
||||
svint32_t svdot_n_s32_s16(svint32_t, svint16_t, int16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_u32_u16)))
|
||||
svuint32_t svdot_n_u32_u16(svuint32_t, svuint16_t, uint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f32_f16)))
|
||||
svfloat32_t svdot_f32_f16(svfloat32_t, svfloat16_t, svfloat16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_s32_s16)))
|
||||
@@ -7341,6 +7379,12 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcreate2_b)))
|
||||
svboolx2_t svcreate2(svbool_t, svbool_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcreate4_b)))
|
||||
svboolx4_t svcreate4(svbool_t, svbool_t, svbool_t, svbool_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f32_f16)))
|
||||
svfloat32_t svdot(svfloat32_t, svfloat16_t, float16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_s32_s16)))
|
||||
svint32_t svdot(svint32_t, svint16_t, int16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_u32_u16)))
|
||||
svuint32_t svdot(svuint32_t, svuint16_t, uint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f32_f16)))
|
||||
svfloat32_t svdot(svfloat32_t, svfloat16_t, svfloat16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_s32_s16)))
|
||||
@@ -9049,6 +9093,346 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_mf8)))
|
||||
svmfloat8_t svexpand(svbool_t, svmfloat8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_s16)))
|
||||
svint16_t svexpand(svbool_t, svint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_s32)))
|
||||
svint32_t svabal_n_s32(svint32_t, svint16_t, int16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_s64)))
|
||||
svint64_t svabal_n_s64(svint64_t, svint32_t, int32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_s16)))
|
||||
svint16_t svabal_n_s16(svint16_t, svint8_t, int8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_u32)))
|
||||
svuint32_t svabal_n_u32(svuint32_t, svuint16_t, uint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_u64)))
|
||||
svuint64_t svabal_n_u64(svuint64_t, svuint32_t, uint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_u16)))
|
||||
svuint16_t svabal_n_u16(svuint16_t, svuint8_t, uint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_s32)))
|
||||
svint32_t svabal_s32(svint32_t, svint16_t, svint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_s64)))
|
||||
svint64_t svabal_s64(svint64_t, svint32_t, svint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_s16)))
|
||||
svint16_t svabal_s16(svint16_t, svint8_t, svint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_u32)))
|
||||
svuint32_t svabal_u32(svuint32_t, svuint16_t, svuint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_u64)))
|
||||
svuint64_t svabal_u64(svuint64_t, svuint32_t, svuint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_u16)))
|
||||
svuint16_t svabal_u16(svuint16_t, svuint8_t, svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u8)))
|
||||
svuint8_t svaddqp_u8(svuint8_t, svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u32)))
|
||||
svuint32_t svaddqp_u32(svuint32_t, svuint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u64)))
|
||||
svuint64_t svaddqp_u64(svuint64_t, svuint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u16)))
|
||||
svuint16_t svaddqp_u16(svuint16_t, svuint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s8)))
|
||||
svint8_t svaddqp_s8(svint8_t, svint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s32)))
|
||||
svint32_t svaddqp_s32(svint32_t, svint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s64)))
|
||||
svint64_t svaddqp_s64(svint64_t, svint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s16)))
|
||||
svint16_t svaddqp_s16(svint16_t, svint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u8)))
|
||||
svuint8_t svaddsubp_u8(svuint8_t, svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u32)))
|
||||
svuint32_t svaddsubp_u32(svuint32_t, svuint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u64)))
|
||||
svuint64_t svaddsubp_u64(svuint64_t, svuint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u16)))
|
||||
svuint16_t svaddsubp_u16(svuint16_t, svuint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s8)))
|
||||
svint8_t svaddsubp_s8(svint8_t, svint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s32)))
|
||||
svint32_t svaddsubp_s32(svint32_t, svint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s64)))
|
||||
svint64_t svaddsubp_s64(svint64_t, svint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s16)))
|
||||
svint16_t svaddsubp_s16(svint16_t, svint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f64_s32)))
|
||||
svfloat64_t svcvtb_f64_s32(svint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f32_s16)))
|
||||
svfloat32_t svcvtb_f32_s16(svint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f16_s8)))
|
||||
svfloat16_t svcvtb_f16_s8(svint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f64_u32)))
|
||||
svfloat64_t svcvtb_f64_u32(svuint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f32_u16)))
|
||||
svfloat32_t svcvtb_f32_u16(svuint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f16_u8)))
|
||||
svfloat16_t svcvtb_f16_u8(svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_s32_f64_x2)))
|
||||
svint32_t svcvtn_s32_f64_x2(svfloat64x2_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_s16_f32_x2)))
|
||||
svint16_t svcvtn_s16_f32_x2(svfloat32x2_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_s8_f16_x2)))
|
||||
svint8_t svcvtn_s8_f16_x2(svfloat16x2_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_u32_f64_x2)))
|
||||
svuint32_t svcvtn_u32_f64_x2(svfloat64x2_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_u16_f32_x2)))
|
||||
svuint16_t svcvtn_u16_f32_x2(svfloat32x2_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_u8_f16_x2)))
|
||||
svuint8_t svcvtn_u8_f16_x2(svfloat16x2_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f64_s32)))
|
||||
svfloat64_t svcvtt_f64_s32(svint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f32_s16)))
|
||||
svfloat32_t svcvtt_f32_s16(svint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f16_s8)))
|
||||
svfloat16_t svcvtt_f16_s8(svint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f64_u32)))
|
||||
svfloat64_t svcvtt_f64_u32(svuint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f32_u16)))
|
||||
svfloat32_t svcvtt_f32_u16(svuint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f16_u8)))
|
||||
svfloat16_t svcvtt_f16_u8(svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_s16_s8)))
|
||||
svint16_t svdot_n_s16_s8(svint16_t, svint8_t, int8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_u16_u8)))
|
||||
svuint16_t svdot_n_u16_u8(svuint16_t, svuint8_t, uint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_s16_s8)))
|
||||
svint16_t svdot_s16_s8(svint16_t, svint8_t, svint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_u16_u8)))
|
||||
svuint16_t svdot_u16_u8(svuint16_t, svuint8_t, svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_s16_s8)))
|
||||
svint16_t svdot_lane_s16_s8(svint16_t, svint8_t, svint8_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_u16_u8)))
|
||||
svuint16_t svdot_lane_u16_u8(svuint16_t, svuint8_t, svuint8_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqrshrn_n_s8_s16_x2)))
|
||||
svint8_t svqrshrn_n_s8_s16_x2(svint16x2_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqrshrn_n_u8_u16_x2)))
|
||||
svuint8_t svqrshrn_n_u8_u16_x2(svuint16x2_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqrshrun_n_u8_s16_x2)))
|
||||
svuint8_t svqrshrun_n_u8_s16_x2(svint16x2_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_s16_s32_x2)))
|
||||
svint16_t svqshrn_n_s16_s32_x2(svint32x2_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_s8_s16_x2)))
|
||||
svint8_t svqshrn_n_s8_s16_x2(svint16x2_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_u16_u32_x2)))
|
||||
svuint16_t svqshrn_n_u16_u32_x2(svuint32x2_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_u8_u16_x2)))
|
||||
svuint8_t svqshrn_n_u8_u16_x2(svuint16x2_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrun_n_u16_s32_x2)))
|
||||
svuint16_t svqshrun_n_u16_s32_x2(svint32x2_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrun_n_u8_s16_x2)))
|
||||
svuint8_t svqshrun_n_u8_s16_x2(svint16x2_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u8_m)))
|
||||
svuint8_t svsubp_u8_m(svbool_t, svuint8_t, svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u32_m)))
|
||||
svuint32_t svsubp_u32_m(svbool_t, svuint32_t, svuint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u64_m)))
|
||||
svuint64_t svsubp_u64_m(svbool_t, svuint64_t, svuint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u16_m)))
|
||||
svuint16_t svsubp_u16_m(svbool_t, svuint16_t, svuint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s8_m)))
|
||||
svint8_t svsubp_s8_m(svbool_t, svint8_t, svint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s32_m)))
|
||||
svint32_t svsubp_s32_m(svbool_t, svint32_t, svint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s64_m)))
|
||||
svint64_t svsubp_s64_m(svbool_t, svint64_t, svint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s16_m)))
|
||||
svint16_t svsubp_s16_m(svbool_t, svint16_t, svint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u8_x)))
|
||||
svuint8_t svsubp_u8_x(svbool_t, svuint8_t, svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u32_x)))
|
||||
svuint32_t svsubp_u32_x(svbool_t, svuint32_t, svuint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u64_x)))
|
||||
svuint64_t svsubp_u64_x(svbool_t, svuint64_t, svuint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u16_x)))
|
||||
svuint16_t svsubp_u16_x(svbool_t, svuint16_t, svuint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s8_x)))
|
||||
svint8_t svsubp_s8_x(svbool_t, svint8_t, svint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s32_x)))
|
||||
svint32_t svsubp_s32_x(svbool_t, svint32_t, svint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s64_x)))
|
||||
svint64_t svsubp_s64_x(svbool_t, svint64_t, svint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s16_x)))
|
||||
svint16_t svsubp_s16_x(svbool_t, svint16_t, svint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u8_z)))
|
||||
svuint8_t svsubp_u8_z(svbool_t, svuint8_t, svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u32_z)))
|
||||
svuint32_t svsubp_u32_z(svbool_t, svuint32_t, svuint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u64_z)))
|
||||
svuint64_t svsubp_u64_z(svbool_t, svuint64_t, svuint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u16_z)))
|
||||
svuint16_t svsubp_u16_z(svbool_t, svuint16_t, svuint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s8_z)))
|
||||
svint8_t svsubp_s8_z(svbool_t, svint8_t, svint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s32_z)))
|
||||
svint32_t svsubp_s32_z(svbool_t, svint32_t, svint32_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s64_z)))
|
||||
svint64_t svsubp_s64_z(svbool_t, svint64_t, svint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s16_z)))
|
||||
svint16_t svsubp_s16_z(svbool_t, svint16_t, svint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_s32)))
|
||||
svint32_t svabal(svint32_t, svint16_t, int16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_s64)))
|
||||
svint64_t svabal(svint64_t, svint32_t, int32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_s16)))
|
||||
svint16_t svabal(svint16_t, svint8_t, int8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_u32)))
|
||||
svuint32_t svabal(svuint32_t, svuint16_t, uint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_u64)))
|
||||
svuint64_t svabal(svuint64_t, svuint32_t, uint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_u16)))
|
||||
svuint16_t svabal(svuint16_t, svuint8_t, uint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_s32)))
|
||||
svint32_t svabal(svint32_t, svint16_t, svint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_s64)))
|
||||
svint64_t svabal(svint64_t, svint32_t, svint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_s16)))
|
||||
svint16_t svabal(svint16_t, svint8_t, svint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_u32)))
|
||||
svuint32_t svabal(svuint32_t, svuint16_t, svuint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_u64)))
|
||||
svuint64_t svabal(svuint64_t, svuint32_t, svuint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_u16)))
|
||||
svuint16_t svabal(svuint16_t, svuint8_t, svuint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u8)))
|
||||
svuint8_t svaddqp(svuint8_t, svuint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u32)))
|
||||
svuint32_t svaddqp(svuint32_t, svuint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u64)))
|
||||
svuint64_t svaddqp(svuint64_t, svuint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u16)))
|
||||
svuint16_t svaddqp(svuint16_t, svuint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s8)))
|
||||
svint8_t svaddqp(svint8_t, svint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s32)))
|
||||
svint32_t svaddqp(svint32_t, svint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s64)))
|
||||
svint64_t svaddqp(svint64_t, svint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s16)))
|
||||
svint16_t svaddqp(svint16_t, svint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u8)))
|
||||
svuint8_t svaddsubp(svuint8_t, svuint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u32)))
|
||||
svuint32_t svaddsubp(svuint32_t, svuint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u64)))
|
||||
svuint64_t svaddsubp(svuint64_t, svuint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u16)))
|
||||
svuint16_t svaddsubp(svuint16_t, svuint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s8)))
|
||||
svint8_t svaddsubp(svint8_t, svint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s32)))
|
||||
svint32_t svaddsubp(svint32_t, svint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s64)))
|
||||
svint64_t svaddsubp(svint64_t, svint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s16)))
|
||||
svint16_t svaddsubp(svint16_t, svint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f64_s32)))
|
||||
svfloat64_t svcvtb_f64(svint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f32_s16)))
|
||||
svfloat32_t svcvtb_f32(svint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f16_s8)))
|
||||
svfloat16_t svcvtb_f16(svint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f64_u32)))
|
||||
svfloat64_t svcvtb_f64(svuint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f32_u16)))
|
||||
svfloat32_t svcvtb_f32(svuint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f16_u8)))
|
||||
svfloat16_t svcvtb_f16(svuint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_s32_f64_x2)))
|
||||
svint32_t svcvtn_s32(svfloat64x2_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_s16_f32_x2)))
|
||||
svint16_t svcvtn_s16(svfloat32x2_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_s8_f16_x2)))
|
||||
svint8_t svcvtn_s8(svfloat16x2_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_u32_f64_x2)))
|
||||
svuint32_t svcvtn_u32(svfloat64x2_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_u16_f32_x2)))
|
||||
svuint16_t svcvtn_u16(svfloat32x2_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_u8_f16_x2)))
|
||||
svuint8_t svcvtn_u8(svfloat16x2_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f64_s32)))
|
||||
svfloat64_t svcvtt_f64(svint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f32_s16)))
|
||||
svfloat32_t svcvtt_f32(svint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f16_s8)))
|
||||
svfloat16_t svcvtt_f16(svint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f64_u32)))
|
||||
svfloat64_t svcvtt_f64(svuint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f32_u16)))
|
||||
svfloat32_t svcvtt_f32(svuint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f16_u8)))
|
||||
svfloat16_t svcvtt_f16(svuint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_s16_s8)))
|
||||
svint16_t svdot(svint16_t, svint8_t, int8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_u16_u8)))
|
||||
svuint16_t svdot(svuint16_t, svuint8_t, uint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_s16_s8)))
|
||||
svint16_t svdot(svint16_t, svint8_t, svint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_u16_u8)))
|
||||
svuint16_t svdot(svuint16_t, svuint8_t, svuint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_s16_s8)))
|
||||
svint16_t svdot_lane(svint16_t, svint8_t, svint8_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_u16_u8)))
|
||||
svuint16_t svdot_lane(svuint16_t, svuint8_t, svuint8_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqrshrn_n_s8_s16_x2)))
|
||||
svint8_t svqrshrn_s8(svint16x2_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqrshrn_n_u8_u16_x2)))
|
||||
svuint8_t svqrshrn_u8(svuint16x2_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqrshrun_n_u8_s16_x2)))
|
||||
svuint8_t svqrshrun_u8(svint16x2_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_s16_s32_x2)))
|
||||
svint16_t svqshrn_s16(svint32x2_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_s8_s16_x2)))
|
||||
svint8_t svqshrn_s8(svint16x2_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_u16_u32_x2)))
|
||||
svuint16_t svqshrn_u16(svuint32x2_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_u8_u16_x2)))
|
||||
svuint8_t svqshrn_u8(svuint16x2_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrun_n_u16_s32_x2)))
|
||||
svuint16_t svqshrun_u16(svint32x2_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrun_n_u8_s16_x2)))
|
||||
svuint8_t svqshrun_u8(svint16x2_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u8_m)))
|
||||
svuint8_t svsubp_m(svbool_t, svuint8_t, svuint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u32_m)))
|
||||
svuint32_t svsubp_m(svbool_t, svuint32_t, svuint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u64_m)))
|
||||
svuint64_t svsubp_m(svbool_t, svuint64_t, svuint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u16_m)))
|
||||
svuint16_t svsubp_m(svbool_t, svuint16_t, svuint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s8_m)))
|
||||
svint8_t svsubp_m(svbool_t, svint8_t, svint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s32_m)))
|
||||
svint32_t svsubp_m(svbool_t, svint32_t, svint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s64_m)))
|
||||
svint64_t svsubp_m(svbool_t, svint64_t, svint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s16_m)))
|
||||
svint16_t svsubp_m(svbool_t, svint16_t, svint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u8_x)))
|
||||
svuint8_t svsubp_x(svbool_t, svuint8_t, svuint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u32_x)))
|
||||
svuint32_t svsubp_x(svbool_t, svuint32_t, svuint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u64_x)))
|
||||
svuint64_t svsubp_x(svbool_t, svuint64_t, svuint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u16_x)))
|
||||
svuint16_t svsubp_x(svbool_t, svuint16_t, svuint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s8_x)))
|
||||
svint8_t svsubp_x(svbool_t, svint8_t, svint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s32_x)))
|
||||
svint32_t svsubp_x(svbool_t, svint32_t, svint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s64_x)))
|
||||
svint64_t svsubp_x(svbool_t, svint64_t, svint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s16_x)))
|
||||
svint16_t svsubp_x(svbool_t, svint16_t, svint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u8_z)))
|
||||
svuint8_t svsubp_z(svbool_t, svuint8_t, svuint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u32_z)))
|
||||
svuint32_t svsubp_z(svbool_t, svuint32_t, svuint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u64_z)))
|
||||
svuint64_t svsubp_z(svbool_t, svuint64_t, svuint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u16_z)))
|
||||
svuint16_t svsubp_z(svbool_t, svuint16_t, svuint16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s8_z)))
|
||||
svint8_t svsubp_z(svbool_t, svint8_t, svint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s32_z)))
|
||||
svint32_t svsubp_z(svbool_t, svint32_t, svint32_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s64_z)))
|
||||
svint64_t svsubp_z(svbool_t, svint64_t, svint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s16_z)))
|
||||
svint16_t svsubp_z(svbool_t, svint16_t, svint16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaba_n_s8)))
|
||||
svint8_t svaba_n_s8(svint8_t, svint8_t, int8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaba_n_s32)))
|
||||
@@ -16581,6 +16965,10 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_x)))
|
||||
svbfloat16_t svsub_x(svbool_t, svbfloat16_t, svbfloat16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_z)))
|
||||
svbfloat16_t svsub_z(svbool_t, svbfloat16_t, svbfloat16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_bf16)))
|
||||
svbfloat16_t svmmla_bf16(svbfloat16_t, svbfloat16_t, svbfloat16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_bf16)))
|
||||
svbfloat16_t svmmla(svbfloat16_t, svbfloat16_t, svbfloat16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_n_bf16_m)))
|
||||
svbfloat16_t svscale_n_bf16_m(svbool_t, svbfloat16_t, int16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_n_bf16_x)))
|
||||
@@ -19415,6 +19803,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c64_s64))
|
||||
svcount_t svwhilelt_c64(int64_t, int64_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c16_s64)))
|
||||
svcount_t svwhilelt_c16(int64_t, int64_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f16)))
|
||||
svfloat16_t svmmla_f16(svfloat16_t, svfloat16_t, svfloat16_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f16)))
|
||||
svfloat16_t svmmla(svfloat16_t, svfloat16_t, svfloat16_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_u8_x2)))
|
||||
svuint8_t svluti6_u8_x2(svuint8x2_t, svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_s8_x2)))
|
||||
svint8_t svluti6_s8_x2(svint8x2_t, svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_mf8_x2)))
|
||||
svmfloat8_t svluti6_mf8_x2(svmfloat8x2_t, svuint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_u8_x2)))
|
||||
svuint8_t svluti6(svuint8x2_t, svuint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_s8_x2)))
|
||||
svint8_t svluti6(svint8x2_t, svuint8_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_mf8_x2)))
|
||||
svmfloat8_t svluti6(svmfloat8x2_t, svuint8_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_u16_x2)))
|
||||
svuint16_t svluti6_lane_u16_x2(svuint16x2_t, svuint8_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_bf16_x2)))
|
||||
svbfloat16_t svluti6_lane_bf16_x2(svbfloat16x2_t, svuint8_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_f16_x2)))
|
||||
svfloat16_t svluti6_lane_f16_x2(svfloat16x2_t, svuint8_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_s16_x2)))
|
||||
svint16_t svluti6_lane_s16_x2(svint16x2_t, svuint8_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_u16_x2)))
|
||||
svuint16_t svluti6_lane(svuint16x2_t, svuint8_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_bf16_x2)))
|
||||
svbfloat16_t svluti6_lane(svbfloat16x2_t, svuint8_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_f16_x2)))
|
||||
svfloat16_t svluti6_lane(svfloat16x2_t, svuint8_t, uint64_t);
|
||||
__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_s16_x2)))
|
||||
svint16_t svluti6_lane(svint16x2_t, svuint8_t, uint64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabd_n_f64_m)))
|
||||
svfloat64_t svabd_n_f64_m(svbool_t, svfloat64_t, float64_t);
|
||||
__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabd_n_f32_m)))
|
||||
|
||||
Vendored
+4
-5
@@ -1810,10 +1810,9 @@ _mm256_or_si256(__m256i __a, __m256i __b)
|
||||
/// \param __b
|
||||
/// A 256-bit integer vector.
|
||||
/// \returns A 256-bit integer vector containing the result.
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_sad_epu8(__m256i __a, __m256i __b)
|
||||
{
|
||||
return __builtin_ia32_psadbw256((__v32qi)__a, (__v32qi)__b);
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_sad_epu8(__m256i __a, __m256i __b) {
|
||||
return __builtin_ia32_psadbw256((__v32qu)__a, (__v32qu)__b);
|
||||
}
|
||||
|
||||
/// Shuffles 8-bit integers in the 256-bit integer vector \a __a according
|
||||
@@ -3058,7 +3057,7 @@ _mm256_broadcastsi128_si256(__m128i __X) {
|
||||
/// An immediate 8-bit integer operand, with bits [7:0] specifying the
|
||||
/// source for each element of the result. The position of the mask bit
|
||||
/// corresponds to the index of a copied value. When a mask bit is 0, the
|
||||
/// element is copied from \a V1; otherwise, it is is copied from \a V2.
|
||||
/// element is copied from \a V1; otherwise, it is copied from \a V2.
|
||||
/// \returns A 256-bit vector of [8 x i32] containing the result.
|
||||
#define _mm256_blend_epi32(V1, V2, M) \
|
||||
((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(V1), \
|
||||
|
||||
Vendored
+174
@@ -0,0 +1,174 @@
|
||||
/*===-------- avx512bmmintrin.h - AVX512BMM intrinsics *------------------===
|
||||
*
|
||||
*
|
||||
* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
* See https://llvm.org/LICENSE.txt for license information.
|
||||
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
*
|
||||
*===---------------------------------------------------------------------===
|
||||
*/
|
||||
|
||||
#ifndef __IMMINTRIN_H
|
||||
#error "Never use <avx512bmmintrin.h> directly; include <immintrin.h> instead."
|
||||
#endif
|
||||
|
||||
#ifndef _AVX512BMMINTRIN_H
|
||||
#define _AVX512BMMINTRIN_H
|
||||
|
||||
/* Define the default attributes for the functions in this file. */
|
||||
#define __DEFAULT_FN_ATTRS \
|
||||
__attribute__((__always_inline__, __nodebug__, __target__("avx512bmm"), \
|
||||
__min_vector_width__(512)))
|
||||
|
||||
#if defined(__cplusplus) && (__cplusplus >= 201103L)
|
||||
#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
|
||||
#else
|
||||
#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
|
||||
#endif
|
||||
|
||||
/// Multiplies two 16x16 bit matrices using OR reduction and ORs the product
|
||||
/// into a third 16x16 bit matrix (which is also the destination).
|
||||
///
|
||||
/// For the 512-bit ZMM form, each register contains two 16x16 (256-bit)
|
||||
/// matrices in bits [255:0] and [511:256]. The operation performs:
|
||||
/// \code{.operation}
|
||||
/// for i in 0 to 15
|
||||
/// for j in 0 to 15
|
||||
/// reduction_bit = __C[16*i+j]
|
||||
/// for k in 0 to 15
|
||||
/// reduction_bit |= __A[16*i+k] & __B[16*k+j]
|
||||
/// end for k
|
||||
/// dest[16*i+j] = reduction_bit
|
||||
/// end for j
|
||||
/// end for i
|
||||
/// \endcode
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> VBMACOR16X16X16 </c> instruction.
|
||||
///
|
||||
/// \param __A
|
||||
/// A 512-bit vector containing two 16x16 bit matrices (one per 256-bit
|
||||
/// lane).
|
||||
/// \param __B
|
||||
/// A 512-bit vector containing two 16x16 bit matrices (one per 256-bit
|
||||
/// lane).
|
||||
/// \param __C
|
||||
/// A 512-bit accumulator vector containing the initial values to OR with.
|
||||
/// \returns A 512-bit vector containing the accumulated result for each lane.
|
||||
/// \note This instruction does not support masking.
|
||||
static __inline __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
_mm512_bmacor16x16x16(__m512i __A, __m512i __B, __m512i __C) {
|
||||
return (__m512i)__builtin_ia32_bmacor16x16x16_v32hi(
|
||||
(__v32hi)__A, (__v32hi)__B, (__v32hi)__C);
|
||||
}
|
||||
|
||||
/// Multiplies two 16x16 bit matrices using XOR reduction and XORs the product
|
||||
/// into a third 16x16 bit matrix (which is also the destination).
|
||||
///
|
||||
/// For the 512-bit ZMM form, each register contains two 16x16 (256-bit)
|
||||
/// matrices in bits [255:0] and [511:256]. The operation performs:
|
||||
/// \code{.operation}
|
||||
/// for i in 0 to 15
|
||||
/// for j in 0 to 15
|
||||
/// reduction_bit = __C[16*i+j]
|
||||
/// for k in 0 to 15
|
||||
/// reduction_bit ^= __A[16*i+k] & __B[16*k+j]
|
||||
/// end for k
|
||||
/// dest[16*i+j] = reduction_bit
|
||||
/// end for j
|
||||
/// end for i
|
||||
/// \endcode
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> VBMACXOR16X16X16 </c> instruction.
|
||||
///
|
||||
/// \param __A
|
||||
/// A 512-bit vector containing two 16x16 bit matrices (one per 256-bit
|
||||
/// lane).
|
||||
/// \param __B
|
||||
/// A 512-bit vector containing two 16x16 bit matrices (one per 256-bit
|
||||
/// lane).
|
||||
/// \param __C
|
||||
/// A 512-bit accumulator vector containing the initial values to XOR with.
|
||||
/// \returns A 512-bit vector containing the accumulated result for each lane.
|
||||
/// \note This instruction does not support masking.
|
||||
static __inline __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
_mm512_bmacxor16x16x16(__m512i __A, __m512i __B, __m512i __C) {
|
||||
return (__m512i)__builtin_ia32_bmacxor16x16x16_v32hi(
|
||||
(__v32hi)__A, (__v32hi)__B, (__v32hi)__C);
|
||||
}
|
||||
|
||||
/// Reverses the bits within each byte of the source vector.
|
||||
///
|
||||
/// For each byte in the source, reverses the order of its 8 bits to generate
|
||||
/// the corresponding destination byte. For example, 0b10110001 becomes
|
||||
/// 0b10001101.
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> VBITREV </c> instruction.
|
||||
///
|
||||
/// \param __A
|
||||
/// A 512-bit vector of [64 x i8] where each byte will have its bits
|
||||
/// reversed.
|
||||
/// \returns A 512-bit vector of [64 x i8] with bit-reversed bytes.
|
||||
static __inline __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
_mm512_bitrev_epi8(__m512i __A) {
|
||||
return (__m512i)__builtin_elementwise_bitreverse((__v64qi)__A);
|
||||
}
|
||||
|
||||
/// Reverses the bits within each byte of the source vector, using a writemask
|
||||
/// to conditionally select elements.
|
||||
///
|
||||
/// For each byte position, if the corresponding mask bit is 1, the byte from
|
||||
/// \a A has its bits reversed and stored in the result. If the mask bit is 0,
|
||||
/// the corresponding byte from \a B is copied to the result (merge masking).
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> VBITREV </c> instruction.
|
||||
///
|
||||
/// \param __U
|
||||
/// A 64-bit mask value where each bit controls one byte (per 8-bit element).
|
||||
/// A 1 performs bit reversal; a 0 selects the passthrough byte from __B.
|
||||
/// \param __A
|
||||
/// A 512-bit vector of [64 x i8] to be bit-reversed.
|
||||
/// \param __B
|
||||
/// A 512-bit vector of [64 x i8] providing passthrough values.
|
||||
/// \returns A 512-bit vector combining bit-reversed and passthrough bytes.
|
||||
static __inline __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
_mm512_mask_bitrev_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
|
||||
return (__m512i)__builtin_ia32_selectb_512(
|
||||
(__mmask64)__U, (__v64qi)_mm512_bitrev_epi8(__A), (__v64qi)__B);
|
||||
}
|
||||
|
||||
/// Reverses the bits within each byte of the source vector, zeroing elements
|
||||
/// based on the writemask.
|
||||
///
|
||||
/// For each byte position, if the corresponding mask bit is 1, the byte from
|
||||
/// \a A has its bits reversed and stored in the result. If the mask bit is 0,
|
||||
/// the result byte is set to zero (zero masking).
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> VBITREV </c> instruction.
|
||||
///
|
||||
/// \param __U
|
||||
/// A 64-bit mask value where each bit controls one byte (per 8-bit element).
|
||||
/// A 1 performs bit reversal; a 0 sets the byte to zero.
|
||||
/// \param __A
|
||||
/// A 512-bit vector of [64 x i8] to be bit-reversed.
|
||||
/// \returns A 512-bit vector with bit-reversed or zeroed bytes.
|
||||
static __inline __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
_mm512_maskz_bitrev_epi8(__mmask64 __U, __m512i __A) {
|
||||
return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
|
||||
(__v64qi)_mm512_bitrev_epi8(__A),
|
||||
(__v64qi)_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
#undef __DEFAULT_FN_ATTRS
|
||||
#undef __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
|
||||
#endif
|
||||
Vendored
+245
@@ -0,0 +1,245 @@
|
||||
/*===------------- avx512bmmvlintrin.h - BMM intrinsics ------------------===
|
||||
*
|
||||
*
|
||||
* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
* See https://llvm.org/LICENSE.txt for license information.
|
||||
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
*
|
||||
*===-----------------------------------------------------------------------===
|
||||
*/
|
||||
#ifndef __IMMINTRIN_H
|
||||
#error \
|
||||
"Never use <avx512bmmvlintrin.h> directly; include <immintrin.h> instead."
|
||||
#endif
|
||||
|
||||
#ifndef __BMMVLINTRIN_H
|
||||
#define __BMMVLINTRIN_H
|
||||
|
||||
/* Define the default attributes for the functions in this file. */
|
||||
#define __DEFAULT_FN_ATTRS128 \
|
||||
__attribute__((__always_inline__, __nodebug__, \
|
||||
__target__("avx512bmm,avx512vl"), __min_vector_width__(128)))
|
||||
#define __DEFAULT_FN_ATTRS256 \
|
||||
__attribute__((__always_inline__, __nodebug__, \
|
||||
__target__("avx512bmm,avx512vl"), __min_vector_width__(256)))
|
||||
|
||||
#if defined(__cplusplus) && (__cplusplus >= 201103L)
|
||||
#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
|
||||
#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
|
||||
#else
|
||||
#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
|
||||
#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
|
||||
#endif
|
||||
|
||||
/// Multiplies two 16x16 bit matrices using OR reduction and ORs the product
|
||||
/// into a third 16x16 bit matrix (which is also the destination).
|
||||
///
|
||||
/// For the 256-bit YMM form, the source registers/memory each contain a single
|
||||
/// 16x16 (256-bit) matrix in bits [255:0]. The operation performs:
|
||||
/// \code{.operation}
|
||||
/// for i in 0 to 15
|
||||
/// for j in 0 to 15
|
||||
/// reduction_bit = __C[16*i+j]
|
||||
/// for k in 0 to 15
|
||||
/// reduction_bit |= __A[16*i+k] & __B[16*k+j]
|
||||
/// end for k
|
||||
/// dest[16*i+j] = reduction_bit
|
||||
/// end for j
|
||||
/// end for i
|
||||
/// \endcode
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> VBMACOR16X16X16 </c> instruction.
|
||||
///
|
||||
/// \param __A
|
||||
/// A 256-bit vector containing a 16x16 bit matrix.
|
||||
/// \param __B
|
||||
/// A 256-bit vector containing a 16x16 bit matrix.
|
||||
/// \param __C
|
||||
/// A 256-bit accumulator vector containing the initial values to OR with.
|
||||
/// \returns A 256-bit vector containing the accumulated result.
|
||||
/// \note This instruction does not support masking.
|
||||
static __inline __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_bmacor16x16x16(__m256i __A, __m256i __B, __m256i __C) {
|
||||
return (__m256i)__builtin_ia32_bmacor16x16x16_v16hi(
|
||||
(__v16hi)__A, (__v16hi)__B, (__v16hi)__C);
|
||||
}
|
||||
|
||||
/// Multiplies two 16x16 bit matrices using XOR reduction and XORs the product
|
||||
/// into a third 16x16 bit matrix (which is also the destination).
|
||||
///
|
||||
/// For the 256-bit YMM form, the source registers/memory each contain a single
|
||||
/// 16x16 (256-bit) matrix in bits [255:0]. The operation performs:
|
||||
/// \code{.operation}
|
||||
/// for i in 0 to 15
|
||||
/// for j in 0 to 15
|
||||
/// reduction_bit = __C[16*i+j]
|
||||
/// for k in 0 to 15
|
||||
/// reduction_bit ^= __A[16*i+k] & __B[16*k+j]
|
||||
/// end for k
|
||||
/// dest[16*i+j] = reduction_bit
|
||||
/// end for j
|
||||
/// end for i
|
||||
/// \endcode
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> VBMACXOR16X16X16 </c> instruction.
|
||||
///
|
||||
/// \param __A
|
||||
/// A 256-bit vector containing a 16x16 bit matrix.
|
||||
/// \param __B
|
||||
/// A 256-bit vector containing a 16x16 bit matrix.
|
||||
/// \param __C
|
||||
/// A 256-bit accumulator vector containing the initial values to XOR with.
|
||||
/// \returns A 256-bit vector containing the accumulated result.
|
||||
/// \note This instruction does not support masking.
|
||||
static __inline __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_bmacxor16x16x16(__m256i __A, __m256i __B, __m256i __C) {
|
||||
return (__m256i)__builtin_ia32_bmacxor16x16x16_v16hi(
|
||||
(__v16hi)__A, (__v16hi)__B, (__v16hi)__C);
|
||||
}
|
||||
|
||||
/// Reverses the bits within each byte of the source vector.
|
||||
///
|
||||
/// For each byte in the source, reverses the order of its 8 bits to generate
|
||||
/// the corresponding destination byte. For example, 0b10110001 becomes
|
||||
/// 0b10001101.
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> VBITREV </c> instruction.
|
||||
///
|
||||
/// \param __A
|
||||
/// A 128-bit vector of [16 x i8] where each byte will have its bits
|
||||
/// reversed.
|
||||
/// \returns A 128-bit vector of [16 x i8] with bit-reversed bytes.
|
||||
static __inline __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm128_bitrev_epi8(__m128i __A) {
|
||||
return (__m128i)__builtin_elementwise_bitreverse((__v16qi)__A);
|
||||
}
|
||||
|
||||
/// Reverses the bits within each byte of the source vector.
|
||||
///
|
||||
/// For each byte in the source, reverses the order of its 8 bits to generate
|
||||
/// the corresponding destination byte. For example, 0b10110001 becomes
|
||||
/// 0b10001101.
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> VBITREV </c> instruction.
|
||||
///
|
||||
/// \param __A
|
||||
/// A 256-bit vector of [32 x i8] where each byte will have its bits
|
||||
/// reversed.
|
||||
/// \returns A 256-bit vector of [32 x i8] with bit-reversed bytes.
|
||||
static __inline __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_bitrev_epi8(__m256i __A) {
|
||||
return (__m256i)__builtin_elementwise_bitreverse((__v32qi)__A);
|
||||
}
|
||||
|
||||
/// Reverses the bits within each byte of the source vector, using a writemask
|
||||
/// to conditionally select elements.
|
||||
///
|
||||
/// For each byte position, if the corresponding mask bit is 1, the byte from
|
||||
/// \a A has its bits reversed and stored in the result. If the mask bit is 0,
|
||||
/// the corresponding byte from \a B is copied to the result (merge masking).
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> VBITREV </c> instruction.
|
||||
///
|
||||
/// \param __U
|
||||
/// A 16-bit mask value where each bit controls one byte (per 8-bit element).
|
||||
/// A 1 performs bit reversal; a 0 selects the passthrough byte from __B.
|
||||
/// \param __A
|
||||
/// A 128-bit vector of [16 x i8] to be bit-reversed.
|
||||
/// \param __B
|
||||
/// A 128-bit vector of [16 x i8] providing passthrough values.
|
||||
/// \returns A 128-bit vector combining bit-reversed and passthrough bytes.
|
||||
static __inline __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm128_mask_bitrev_epi8(__mmask16 __U, __m128i __A, __m128i __B) {
|
||||
return (__m128i)__builtin_ia32_selectb_128(
|
||||
(__mmask16)__U, (__v16qi)_mm128_bitrev_epi8(__A), (__v16qi)__B);
|
||||
}
|
||||
|
||||
/// Reverses the bits within each byte of the source vector, using a writemask
|
||||
/// to conditionally select elements.
|
||||
///
|
||||
/// For each byte position, if the corresponding mask bit is 1, the byte from
|
||||
/// \a A has its bits reversed and stored in the result. If the mask bit is 0,
|
||||
/// the corresponding byte from \a B is copied to the result (merge masking).
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> VBITREV </c> instruction.
|
||||
///
|
||||
/// \param __U
|
||||
/// A 32-bit mask value where each bit controls one byte (per 8-bit element).
|
||||
/// A 1 performs bit reversal; a 0 selects the passthrough byte from __B.
|
||||
/// \param __A
|
||||
/// A 256-bit vector of [32 x i8] to be bit-reversed.
|
||||
/// \param __B
|
||||
/// A 256-bit vector of [32 x i8] providing passthrough values.
|
||||
/// \returns A 256-bit vector combining bit-reversed and passthrough bytes.
|
||||
static __inline __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_bitrev_epi8(__mmask32 __U, __m256i __A, __m256i __B) {
|
||||
return (__m256i)__builtin_ia32_selectb_256(
|
||||
(__mmask32)__U, (__v32qi)_mm256_bitrev_epi8(__A), (__v32qi)__B);
|
||||
}
|
||||
|
||||
/// Reverses the bits within each byte of the source vector, zeroing elements
|
||||
/// based on the writemask.
|
||||
///
|
||||
/// For each byte position, if the corresponding mask bit is 1, the byte from
|
||||
/// \a A has its bits reversed and stored in the result. If the mask bit is 0,
|
||||
/// the result byte is set to zero (zero masking).
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> VBITREV </c> instruction.
|
||||
///
|
||||
/// \param __U
|
||||
/// A 16-bit mask value where each bit controls one byte (per 8-bit element).
|
||||
/// A 1 performs bit reversal; a 0 sets the byte to zero.
|
||||
/// \param __A
|
||||
/// A 128-bit vector of [16 x i8] to be bit-reversed.
|
||||
/// \returns A 128-bit vector with bit-reversed or zeroed bytes.
|
||||
static __inline __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm128_maskz_bitrev_epi8(__mmask16 __U, __m128i __A) {
|
||||
return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
|
||||
(__v16qi)_mm128_bitrev_epi8(__A),
|
||||
(__v16qi)_mm_setzero_si128());
|
||||
}
|
||||
|
||||
/// Reverses the bits within each byte of the source vector, zeroing elements
|
||||
/// based on the writemask.
|
||||
///
|
||||
/// For each byte position, if the corresponding mask bit is 1, the byte from
|
||||
/// \a A has its bits reversed and stored in the result. If the mask bit is 0,
|
||||
/// the result byte is set to zero (zero masking).
|
||||
///
|
||||
/// \headerfile <immintrin.h>
|
||||
///
|
||||
/// This intrinsic corresponds to the <c> VBITREV </c> instruction.
|
||||
///
|
||||
/// \param __U
|
||||
/// A 32-bit mask value where each bit controls one byte (per 8-bit element).
|
||||
/// A 1 performs bit reversal; a 0 sets the byte to zero.
|
||||
/// \param __A
|
||||
/// A 256-bit vector of [32 x i8] to be bit-reversed.
|
||||
/// \returns A 256-bit vector with bit-reversed or zeroed bytes.
|
||||
static __inline __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_maskz_bitrev_epi8(__mmask32 __U, __m256i __A) {
|
||||
return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
|
||||
(__v32qi)_mm256_bitrev_epi8(__A),
|
||||
(__v32qi)_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
#undef __DEFAULT_FN_ATTRS128
|
||||
#undef __DEFAULT_FN_ATTRS256
|
||||
|
||||
#endif
|
||||
Vendored
+19
-29
@@ -1695,57 +1695,49 @@ _mm512_mask_storeu_epi8 (void *__P, __mmask64 __U, __m512i __A)
|
||||
(__mmask64) __U);
|
||||
}
|
||||
|
||||
static __inline__ __mmask64 __DEFAULT_FN_ATTRS512
|
||||
_mm512_test_epi8_mask (__m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __mmask64 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_test_epi8_mask(__m512i __A, __m512i __B) {
|
||||
return _mm512_cmpneq_epi8_mask (_mm512_and_epi32 (__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __mmask64 __DEFAULT_FN_ATTRS512
|
||||
_mm512_mask_test_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __mmask64 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_test_epi8_mask(__mmask64 __U, __m512i __A, __m512i __B) {
|
||||
return _mm512_mask_cmpneq_epi8_mask (__U, _mm512_and_epi32 (__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __mmask32 __DEFAULT_FN_ATTRS512
|
||||
_mm512_test_epi16_mask (__m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __mmask32 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_test_epi16_mask(__m512i __A, __m512i __B) {
|
||||
return _mm512_cmpneq_epi16_mask (_mm512_and_epi32 (__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __mmask32 __DEFAULT_FN_ATTRS512
|
||||
_mm512_mask_test_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __mmask32 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_test_epi16_mask(__mmask32 __U, __m512i __A, __m512i __B) {
|
||||
return _mm512_mask_cmpneq_epi16_mask (__U, _mm512_and_epi32 (__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __mmask64 __DEFAULT_FN_ATTRS512
|
||||
_mm512_testn_epi8_mask (__m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __mmask64 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_testn_epi8_mask(__m512i __A, __m512i __B) {
|
||||
return _mm512_cmpeq_epi8_mask (_mm512_and_epi32 (__A, __B), _mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __mmask64 __DEFAULT_FN_ATTRS512
|
||||
_mm512_mask_testn_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __mmask64 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_testn_epi8_mask(__mmask64 __U, __m512i __A, __m512i __B) {
|
||||
return _mm512_mask_cmpeq_epi8_mask (__U, _mm512_and_epi32 (__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __mmask32 __DEFAULT_FN_ATTRS512
|
||||
_mm512_testn_epi16_mask (__m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __mmask32 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_testn_epi16_mask(__m512i __A, __m512i __B) {
|
||||
return _mm512_cmpeq_epi16_mask (_mm512_and_epi32 (__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __mmask32 __DEFAULT_FN_ATTRS512
|
||||
_mm512_mask_testn_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __mmask32 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_testn_epi16_mask(__mmask32 __U, __m512i __A, __m512i __B) {
|
||||
return _mm512_mask_cmpeq_epi16_mask (__U, _mm512_and_epi32 (__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
@@ -1880,11 +1872,9 @@ _mm512_mask_permutexvar_epi16(__m512i __W, __mmask32 __M, __m512i __A,
|
||||
(__v32hi)_mm512_dbsad_epu8((A), (B), (imm)), \
|
||||
(__v32hi)_mm512_setzero_si512()))
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512
|
||||
_mm512_sad_epu8 (__m512i __A, __m512i __B)
|
||||
{
|
||||
return (__m512i) __builtin_ia32_psadbw512 ((__v64qi) __A,
|
||||
(__v64qi) __B);
|
||||
static __inline__ __m512i
|
||||
__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_sad_epu8(__m512i __A, __m512i __B) {
|
||||
return (__m512i)__builtin_ia32_psadbw512((__v64qu)__A, (__v64qu)__B);
|
||||
}
|
||||
|
||||
#undef __DEFAULT_FN_ATTRS512
|
||||
|
||||
Vendored
+62
-89
@@ -42,11 +42,12 @@ typedef unsigned char __mmask8;
|
||||
typedef unsigned short __mmask16;
|
||||
|
||||
/* Rounding mode macros. */
|
||||
#define _MM_FROUND_TO_NEAREST_INT 0x00
|
||||
#define _MM_FROUND_TO_NEG_INF 0x01
|
||||
#define _MM_FROUND_TO_POS_INF 0x02
|
||||
#define _MM_FROUND_TO_ZERO 0x03
|
||||
#define _MM_FROUND_CUR_DIRECTION 0x04
|
||||
#define _MM_FROUND_TO_NEAREST_INT 0x00
|
||||
#define _MM_FROUND_TO_NEAREST_TIES_EVEN 0x00
|
||||
#define _MM_FROUND_TO_NEG_INF 0x01
|
||||
#define _MM_FROUND_TO_POS_INF 0x02
|
||||
#define _MM_FROUND_TO_ZERO 0x03
|
||||
#define _MM_FROUND_CUR_DIRECTION 0x04
|
||||
|
||||
/* Constants for integer comparison predicates */
|
||||
typedef enum {
|
||||
@@ -986,10 +987,8 @@ _mm512_maskz_max_ps(__mmask16 __U, __m512 __A, __m512 __B) {
|
||||
(__v16sf)_mm512_setzero_ps());
|
||||
}
|
||||
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_mask_max_ss(__m128 __W,
|
||||
__mmask8 __U,
|
||||
__m128 __A,
|
||||
__m128 __B) {
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_max_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
|
||||
return (__m128) __builtin_ia32_maxss_round_mask ((__v4sf) __A,
|
||||
(__v4sf) __B,
|
||||
(__v4sf) __W,
|
||||
@@ -997,9 +996,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_mask_max_ss(__m128 __W,
|
||||
_MM_FROUND_CUR_DIRECTION);
|
||||
}
|
||||
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_maskz_max_ss(__mmask8 __U,
|
||||
__m128 __A,
|
||||
__m128 __B) {
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_max_ss(__mmask8 __U, __m128 __A, __m128 __B) {
|
||||
return (__m128) __builtin_ia32_maxss_round_mask ((__v4sf) __A,
|
||||
(__v4sf) __B,
|
||||
(__v4sf) _mm_setzero_ps (),
|
||||
@@ -1025,10 +1023,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_maskz_max_ss(__mmask8 __U,
|
||||
(__v4sf)_mm_setzero_ps(), \
|
||||
(__mmask8)(U), (int)(R)))
|
||||
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_mask_max_sd(__m128d __W,
|
||||
__mmask8 __U,
|
||||
__m128d __A,
|
||||
__m128d __B) {
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_max_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
|
||||
return (__m128d) __builtin_ia32_maxsd_round_mask ((__v2df) __A,
|
||||
(__v2df) __B,
|
||||
(__v2df) __W,
|
||||
@@ -1036,9 +1032,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_mask_max_sd(__m128d __W,
|
||||
_MM_FROUND_CUR_DIRECTION);
|
||||
}
|
||||
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_maskz_max_sd(__mmask8 __U,
|
||||
__m128d __A,
|
||||
__m128d __B) {
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_max_sd(__mmask8 __U, __m128d __A, __m128d __B) {
|
||||
return (__m128d) __builtin_ia32_maxsd_round_mask ((__v2df) __A,
|
||||
(__v2df) __B,
|
||||
(__v2df) _mm_setzero_pd (),
|
||||
@@ -1208,10 +1203,8 @@ _mm512_maskz_min_ps(__mmask16 __U, __m512 __A, __m512 __B) {
|
||||
(__v16sf)_mm512_setzero_ps());
|
||||
}
|
||||
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_mask_min_ss(__m128 __W,
|
||||
__mmask8 __U,
|
||||
__m128 __A,
|
||||
__m128 __B) {
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_min_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
|
||||
return (__m128) __builtin_ia32_minss_round_mask ((__v4sf) __A,
|
||||
(__v4sf) __B,
|
||||
(__v4sf) __W,
|
||||
@@ -1219,9 +1212,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_mask_min_ss(__m128 __W,
|
||||
_MM_FROUND_CUR_DIRECTION);
|
||||
}
|
||||
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_maskz_min_ss(__mmask8 __U,
|
||||
__m128 __A,
|
||||
__m128 __B) {
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_min_ss(__mmask8 __U, __m128 __A, __m128 __B) {
|
||||
return (__m128) __builtin_ia32_minss_round_mask ((__v4sf) __A,
|
||||
(__v4sf) __B,
|
||||
(__v4sf) _mm_setzero_ps (),
|
||||
@@ -1247,10 +1239,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_maskz_min_ss(__mmask8 __U,
|
||||
(__v4sf)_mm_setzero_ps(), \
|
||||
(__mmask8)(U), (int)(R)))
|
||||
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_mask_min_sd(__m128d __W,
|
||||
__mmask8 __U,
|
||||
__m128d __A,
|
||||
__m128d __B) {
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_min_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
|
||||
return (__m128d) __builtin_ia32_minsd_round_mask ((__v2df) __A,
|
||||
(__v2df) __B,
|
||||
(__v2df) __W,
|
||||
@@ -1258,9 +1248,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_mask_min_sd(__m128d __W,
|
||||
_MM_FROUND_CUR_DIRECTION);
|
||||
}
|
||||
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_maskz_min_sd(__mmask8 __U,
|
||||
__m128d __A,
|
||||
__m128d __B) {
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_min_sd(__mmask8 __U, __m128d __A, __m128d __B) {
|
||||
return (__m128d) __builtin_ia32_minsd_round_mask ((__v2df) __A,
|
||||
(__v2df) __B,
|
||||
(__v2df) _mm_setzero_pd (),
|
||||
@@ -8108,68 +8097,60 @@ _mm512_stream_ps (void *__P, __m512 __A)
|
||||
__builtin_nontemporal_store((__v16sf_aligned)__A, (__v16sf_aligned*)__P);
|
||||
}
|
||||
|
||||
static __inline__ __m512d __DEFAULT_FN_ATTRS512
|
||||
_mm512_mask_compress_pd (__m512d __W, __mmask8 __U, __m512d __A)
|
||||
{
|
||||
static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_compress_pd(__m512d __W, __mmask8 __U, __m512d __A) {
|
||||
return (__m512d) __builtin_ia32_compressdf512_mask ((__v8df) __A,
|
||||
(__v8df) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m512d __DEFAULT_FN_ATTRS512
|
||||
_mm512_maskz_compress_pd (__mmask8 __U, __m512d __A)
|
||||
{
|
||||
static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_maskz_compress_pd(__mmask8 __U, __m512d __A) {
|
||||
return (__m512d) __builtin_ia32_compressdf512_mask ((__v8df) __A,
|
||||
(__v8df)
|
||||
_mm512_setzero_pd (),
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512
|
||||
_mm512_mask_compress_epi64 (__m512i __W, __mmask8 __U, __m512i __A)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_compress_epi64(__m512i __W, __mmask8 __U, __m512i __A) {
|
||||
return (__m512i) __builtin_ia32_compressdi512_mask ((__v8di) __A,
|
||||
(__v8di) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512
|
||||
_mm512_maskz_compress_epi64 (__mmask8 __U, __m512i __A)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_maskz_compress_epi64(__mmask8 __U, __m512i __A) {
|
||||
return (__m512i) __builtin_ia32_compressdi512_mask ((__v8di) __A,
|
||||
(__v8di)
|
||||
_mm512_setzero_si512 (),
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m512 __DEFAULT_FN_ATTRS512
|
||||
_mm512_mask_compress_ps (__m512 __W, __mmask16 __U, __m512 __A)
|
||||
{
|
||||
static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_compress_ps(__m512 __W, __mmask16 __U, __m512 __A) {
|
||||
return (__m512) __builtin_ia32_compresssf512_mask ((__v16sf) __A,
|
||||
(__v16sf) __W,
|
||||
(__mmask16) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m512 __DEFAULT_FN_ATTRS512
|
||||
_mm512_maskz_compress_ps (__mmask16 __U, __m512 __A)
|
||||
{
|
||||
static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_maskz_compress_ps(__mmask16 __U, __m512 __A) {
|
||||
return (__m512) __builtin_ia32_compresssf512_mask ((__v16sf) __A,
|
||||
(__v16sf)
|
||||
_mm512_setzero_ps (),
|
||||
(__mmask16) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512
|
||||
_mm512_mask_compress_epi32 (__m512i __W, __mmask16 __U, __m512i __A)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_compress_epi32(__m512i __W, __mmask16 __U, __m512i __A) {
|
||||
return (__m512i) __builtin_ia32_compresssi512_mask ((__v16si) __A,
|
||||
(__v16si) __W,
|
||||
(__mmask16) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512
|
||||
_mm512_maskz_compress_epi32 (__mmask16 __U, __m512i __A)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_maskz_compress_epi32(__mmask16 __U, __m512i __A) {
|
||||
return (__m512i) __builtin_ia32_compresssi512_mask ((__v16si) __A,
|
||||
(__v16si)
|
||||
_mm512_setzero_si512 (),
|
||||
@@ -8222,58 +8203,50 @@ _mm512_maskz_compress_epi32 (__mmask16 __U, __m512i __A)
|
||||
|
||||
/* Bit Test */
|
||||
|
||||
static __inline __mmask16 __DEFAULT_FN_ATTRS512
|
||||
_mm512_test_epi32_mask (__m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline __mmask16 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_test_epi32_mask(__m512i __A, __m512i __B) {
|
||||
return _mm512_cmpneq_epi32_mask (_mm512_and_epi32(__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
|
||||
_mm512_mask_test_epi32_mask (__mmask16 __U, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_test_epi32_mask(__mmask16 __U, __m512i __A, __m512i __B) {
|
||||
return _mm512_mask_cmpneq_epi32_mask (__U, _mm512_and_epi32 (__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline __mmask8 __DEFAULT_FN_ATTRS512
|
||||
_mm512_test_epi64_mask (__m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline __mmask8 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_test_epi64_mask(__m512i __A, __m512i __B) {
|
||||
return _mm512_cmpneq_epi64_mask (_mm512_and_epi32 (__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS512
|
||||
_mm512_mask_test_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_test_epi64_mask(__mmask8 __U, __m512i __A, __m512i __B) {
|
||||
return _mm512_mask_cmpneq_epi64_mask (__U, _mm512_and_epi32 (__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
|
||||
_mm512_testn_epi32_mask (__m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_testn_epi32_mask(__m512i __A, __m512i __B) {
|
||||
return _mm512_cmpeq_epi32_mask (_mm512_and_epi32 (__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
|
||||
_mm512_mask_testn_epi32_mask (__mmask16 __U, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_testn_epi32_mask(__mmask16 __U, __m512i __A, __m512i __B) {
|
||||
return _mm512_mask_cmpeq_epi32_mask (__U, _mm512_and_epi32 (__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS512
|
||||
_mm512_testn_epi64_mask (__m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_testn_epi64_mask(__m512i __A, __m512i __B) {
|
||||
return _mm512_cmpeq_epi64_mask (_mm512_and_epi32 (__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS512
|
||||
_mm512_mask_testn_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_testn_epi64_mask(__mmask8 __U, __m512i __A, __m512i __B) {
|
||||
return _mm512_mask_cmpeq_epi64_mask (__U, _mm512_and_epi32 (__A, __B),
|
||||
_mm512_setzero_si512());
|
||||
}
|
||||
@@ -8403,7 +8376,7 @@ _mm_maskz_load_sd (__mmask8 __U, const double* __A)
|
||||
(__v16si)_mm512_shuffle_epi32((A), (I)), \
|
||||
(__v16si)_mm512_setzero_si512()))
|
||||
|
||||
static __inline__ __m512d __DEFAULT_FN_ATTRS512
|
||||
static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_expand_pd (__m512d __W, __mmask8 __U, __m512d __A)
|
||||
{
|
||||
return (__m512d) __builtin_ia32_expanddf512_mask ((__v8df) __A,
|
||||
@@ -8411,7 +8384,7 @@ _mm512_mask_expand_pd (__m512d __W, __mmask8 __U, __m512d __A)
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m512d __DEFAULT_FN_ATTRS512
|
||||
static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_maskz_expand_pd (__mmask8 __U, __m512d __A)
|
||||
{
|
||||
return (__m512d) __builtin_ia32_expanddf512_mask ((__v8df) __A,
|
||||
@@ -8419,7 +8392,7 @@ _mm512_maskz_expand_pd (__mmask8 __U, __m512d __A)
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_expand_epi64 (__m512i __W, __mmask8 __U, __m512i __A)
|
||||
{
|
||||
return (__m512i) __builtin_ia32_expanddi512_mask ((__v8di) __A,
|
||||
@@ -8427,7 +8400,7 @@ _mm512_mask_expand_epi64 (__m512i __W, __mmask8 __U, __m512i __A)
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_maskz_expand_epi64 ( __mmask8 __U, __m512i __A)
|
||||
{
|
||||
return (__m512i) __builtin_ia32_expanddi512_mask ((__v8di) __A,
|
||||
@@ -8499,7 +8472,7 @@ _mm512_maskz_expandloadu_epi32(__mmask16 __U, void const *__P)
|
||||
(__mmask16) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m512 __DEFAULT_FN_ATTRS512
|
||||
static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_expand_ps (__m512 __W, __mmask16 __U, __m512 __A)
|
||||
{
|
||||
return (__m512) __builtin_ia32_expandsf512_mask ((__v16sf) __A,
|
||||
@@ -8507,7 +8480,7 @@ _mm512_mask_expand_ps (__m512 __W, __mmask16 __U, __m512 __A)
|
||||
(__mmask16) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m512 __DEFAULT_FN_ATTRS512
|
||||
static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_maskz_expand_ps (__mmask16 __U, __m512 __A)
|
||||
{
|
||||
return (__m512) __builtin_ia32_expandsf512_mask ((__v16sf) __A,
|
||||
@@ -8515,7 +8488,7 @@ _mm512_maskz_expand_ps (__mmask16 __U, __m512 __A)
|
||||
(__mmask16) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_mask_expand_epi32 (__m512i __W, __mmask16 __U, __m512i __A)
|
||||
{
|
||||
return (__m512i) __builtin_ia32_expandsi512_mask ((__v16si) __A,
|
||||
@@ -8523,7 +8496,7 @@ _mm512_mask_expand_epi32 (__m512i __W, __mmask16 __U, __m512i __A)
|
||||
(__mmask16) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
|
||||
_mm512_maskz_expand_epi32 (__mmask16 __U, __m512i __A)
|
||||
{
|
||||
return (__m512i) __builtin_ia32_expandsi512_mask ((__v16si) __A,
|
||||
|
||||
Vendored
+12
-18
@@ -720,25 +720,22 @@ _mm_maskz_div_sh(__mmask8 __U, __m128h __A, __m128h __B) {
|
||||
(__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (__v8hf)_mm_setzero_ph(), \
|
||||
(__mmask8)(U), (int)(R)))
|
||||
|
||||
static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_min_sh(__m128h __A,
|
||||
__m128h __B) {
|
||||
static __inline__ __m128h
|
||||
__DEFAULT_FN_ATTRS128_CONSTEXPR _mm_min_sh(__m128h __A, __m128h __B) {
|
||||
return (__m128h)__builtin_ia32_minsh_round_mask(
|
||||
(__v8hf)__A, (__v8hf)__B, (__v8hf)_mm_setzero_ph(), (__mmask8)-1,
|
||||
_MM_FROUND_CUR_DIRECTION);
|
||||
}
|
||||
|
||||
static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_min_sh(__m128h __W,
|
||||
__mmask8 __U,
|
||||
__m128h __A,
|
||||
__m128h __B) {
|
||||
static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_min_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
|
||||
return (__m128h)__builtin_ia32_minsh_round_mask((__v8hf)__A, (__v8hf)__B,
|
||||
(__v8hf)__W, (__mmask8)__U,
|
||||
_MM_FROUND_CUR_DIRECTION);
|
||||
}
|
||||
|
||||
static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_min_sh(__mmask8 __U,
|
||||
__m128h __A,
|
||||
__m128h __B) {
|
||||
static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_min_sh(__mmask8 __U, __m128h __A, __m128h __B) {
|
||||
return (__m128h)__builtin_ia32_minsh_round_mask(
|
||||
(__v8hf)__A, (__v8hf)__B, (__v8hf)_mm_setzero_ph(), (__mmask8)__U,
|
||||
_MM_FROUND_CUR_DIRECTION);
|
||||
@@ -759,25 +756,22 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_min_sh(__mmask8 __U,
|
||||
(__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (__v8hf)_mm_setzero_ph(), \
|
||||
(__mmask8)(U), (int)(R)))
|
||||
|
||||
static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_max_sh(__m128h __A,
|
||||
__m128h __B) {
|
||||
static __inline__ __m128h
|
||||
__DEFAULT_FN_ATTRS128_CONSTEXPR _mm_max_sh(__m128h __A, __m128h __B) {
|
||||
return (__m128h)__builtin_ia32_maxsh_round_mask(
|
||||
(__v8hf)__A, (__v8hf)__B, (__v8hf)_mm_setzero_ph(), (__mmask8)-1,
|
||||
_MM_FROUND_CUR_DIRECTION);
|
||||
}
|
||||
|
||||
static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_max_sh(__m128h __W,
|
||||
__mmask8 __U,
|
||||
__m128h __A,
|
||||
__m128h __B) {
|
||||
static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_max_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
|
||||
return (__m128h)__builtin_ia32_maxsh_round_mask((__v8hf)__A, (__v8hf)__B,
|
||||
(__v8hf)__W, (__mmask8)__U,
|
||||
_MM_FROUND_CUR_DIRECTION);
|
||||
}
|
||||
|
||||
static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_max_sh(__mmask8 __U,
|
||||
__m128h __A,
|
||||
__m128h __B) {
|
||||
static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_max_sh(__mmask8 __U, __m128h __A, __m128h __B) {
|
||||
return (__m128h)__builtin_ia32_maxsh_round_mask(
|
||||
(__v8hf)__A, (__v8hf)__B, (__v8hf)_mm_setzero_ph(), (__mmask8)__U,
|
||||
_MM_FROUND_CUR_DIRECTION);
|
||||
|
||||
Vendored
+12
-16
@@ -25,33 +25,29 @@
|
||||
#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
|
||||
#endif
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_mask_compress_epi16(__m512i __S, __mmask32 __U, __m512i __D)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
_mm512_mask_compress_epi16(__m512i __S, __mmask32 __U, __m512i __D) {
|
||||
return (__m512i) __builtin_ia32_compresshi512_mask ((__v32hi) __D,
|
||||
(__v32hi) __S,
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_maskz_compress_epi16(__mmask32 __U, __m512i __D)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
_mm512_maskz_compress_epi16(__mmask32 __U, __m512i __D) {
|
||||
return (__m512i) __builtin_ia32_compresshi512_mask ((__v32hi) __D,
|
||||
(__v32hi) _mm512_setzero_si512(),
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_mask_compress_epi8(__m512i __S, __mmask64 __U, __m512i __D)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
_mm512_mask_compress_epi8(__m512i __S, __mmask64 __U, __m512i __D) {
|
||||
return (__m512i) __builtin_ia32_compressqi512_mask ((__v64qi) __D,
|
||||
(__v64qi) __S,
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_maskz_compress_epi8(__mmask64 __U, __m512i __D)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
_mm512_maskz_compress_epi8(__mmask64 __U, __m512i __D) {
|
||||
return (__m512i) __builtin_ia32_compressqi512_mask ((__v64qi) __D,
|
||||
(__v64qi) _mm512_setzero_si512(),
|
||||
__U);
|
||||
@@ -71,7 +67,7 @@ _mm512_mask_compressstoreu_epi8(void *__P, __mmask64 __U, __m512i __D)
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
_mm512_mask_expand_epi16(__m512i __S, __mmask32 __U, __m512i __D)
|
||||
{
|
||||
return (__m512i) __builtin_ia32_expandhi512_mask ((__v32hi) __D,
|
||||
@@ -79,7 +75,7 @@ _mm512_mask_expand_epi16(__m512i __S, __mmask32 __U, __m512i __D)
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
_mm512_maskz_expand_epi16(__mmask32 __U, __m512i __D)
|
||||
{
|
||||
return (__m512i) __builtin_ia32_expandhi512_mask ((__v32hi) __D,
|
||||
@@ -87,7 +83,7 @@ _mm512_maskz_expand_epi16(__mmask32 __U, __m512i __D)
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
_mm512_mask_expand_epi8(__m512i __S, __mmask64 __U, __m512i __D)
|
||||
{
|
||||
return (__m512i) __builtin_ia32_expandqi512_mask ((__v64qi) __D,
|
||||
@@ -95,7 +91,7 @@ _mm512_mask_expand_epi8(__m512i __S, __mmask64 __U, __m512i __D)
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
_mm512_maskz_expand_epi8(__mmask64 __U, __m512i __D)
|
||||
{
|
||||
return (__m512i) __builtin_ia32_expandqi512_mask ((__v64qi) __D,
|
||||
|
||||
Vendored
+24
-36
@@ -2394,36 +2394,31 @@ _mm256_test_epi8_mask(__m256i __A, __m256i __B) {
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __mmask32 __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_test_epi8_mask (__mmask32 __U, __m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask32 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_test_epi8_mask(__mmask32 __U, __m256i __A, __m256i __B) {
|
||||
return _mm256_mask_cmpneq_epi8_mask (__U, _mm256_and_si256(__A, __B),
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
|
||||
_mm_test_epi16_mask (__m128i __A, __m128i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_test_epi16_mask(__m128i __A, __m128i __B) {
|
||||
return _mm_cmpneq_epi16_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_test_epi16_mask (__mmask8 __U, __m128i __A, __m128i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_test_epi16_mask(__mmask8 __U, __m128i __A, __m128i __B) {
|
||||
return _mm_mask_cmpneq_epi16_mask (__U, _mm_and_si128 (__A, __B),
|
||||
_mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS256
|
||||
_mm256_test_epi16_mask (__m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_test_epi16_mask(__m256i __A, __m256i __B) {
|
||||
return _mm256_cmpneq_epi16_mask (_mm256_and_si256 (__A, __B),
|
||||
_mm256_setzero_si256 ());
|
||||
}
|
||||
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_test_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_test_epi16_mask(__mmask16 __U, __m256i __A, __m256i __B) {
|
||||
return _mm256_mask_cmpneq_epi16_mask (__U, _mm256_and_si256(__A, __B),
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
@@ -2433,49 +2428,42 @@ _mm_testn_epi8_mask(__m128i __A, __m128i __B) {
|
||||
return _mm_cmpeq_epi8_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_testn_epi8_mask (__mmask16 __U, __m128i __A, __m128i __B)
|
||||
{
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_testn_epi8_mask(__mmask16 __U, __m128i __A, __m128i __B) {
|
||||
return _mm_mask_cmpeq_epi8_mask (__U, _mm_and_si128 (__A, __B),
|
||||
_mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __mmask32 __DEFAULT_FN_ATTRS256
|
||||
_mm256_testn_epi8_mask (__m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask32 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_testn_epi8_mask(__m256i __A, __m256i __B) {
|
||||
return _mm256_cmpeq_epi8_mask (_mm256_and_si256 (__A, __B),
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __mmask32 __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_testn_epi8_mask (__mmask32 __U, __m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask32 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_testn_epi8_mask(__mmask32 __U, __m256i __A, __m256i __B) {
|
||||
return _mm256_mask_cmpeq_epi8_mask (__U, _mm256_and_si256 (__A, __B),
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
|
||||
_mm_testn_epi16_mask (__m128i __A, __m128i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_testn_epi16_mask(__m128i __A, __m128i __B) {
|
||||
return _mm_cmpeq_epi16_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_testn_epi16_mask (__mmask8 __U, __m128i __A, __m128i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_testn_epi16_mask(__mmask8 __U, __m128i __A, __m128i __B) {
|
||||
return _mm_mask_cmpeq_epi16_mask (__U, _mm_and_si128(__A, __B), _mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS256
|
||||
_mm256_testn_epi16_mask (__m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_testn_epi16_mask(__m256i __A, __m256i __B) {
|
||||
return _mm256_cmpeq_epi16_mask (_mm256_and_si256(__A, __B),
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_testn_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask16 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_testn_epi16_mask(__mmask16 __U, __m256i __A, __m256i __B) {
|
||||
return _mm256_mask_cmpeq_epi16_mask (__U, _mm256_and_si256 (__A, __B),
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
Vendored
+80
-96
@@ -1530,120 +1530,120 @@ _mm256_mask_blend_epi64(__mmask8 __U, __m256i __A, __m256i __W) {
|
||||
(__v4di) __A);
|
||||
}
|
||||
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_compress_pd (__m128d __W, __mmask8 __U, __m128d __A) {
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_compress_pd(__m128d __W, __mmask8 __U, __m128d __A) {
|
||||
return (__m128d) __builtin_ia32_compressdf128_mask ((__v2df) __A,
|
||||
(__v2df) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128
|
||||
_mm_maskz_compress_pd (__mmask8 __U, __m128d __A) {
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_compress_pd(__mmask8 __U, __m128d __A) {
|
||||
return (__m128d) __builtin_ia32_compressdf128_mask ((__v2df) __A,
|
||||
(__v2df)
|
||||
_mm_setzero_pd (),
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256d __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_compress_pd (__m256d __W, __mmask8 __U, __m256d __A) {
|
||||
static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_compress_pd(__m256d __W, __mmask8 __U, __m256d __A) {
|
||||
return (__m256d) __builtin_ia32_compressdf256_mask ((__v4df) __A,
|
||||
(__v4df) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256d __DEFAULT_FN_ATTRS256
|
||||
_mm256_maskz_compress_pd (__mmask8 __U, __m256d __A) {
|
||||
static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_maskz_compress_pd(__mmask8 __U, __m256d __A) {
|
||||
return (__m256d) __builtin_ia32_compressdf256_mask ((__v4df) __A,
|
||||
(__v4df)
|
||||
_mm256_setzero_pd (),
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_compress_epi64 (__m128i __W, __mmask8 __U, __m128i __A) {
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_compress_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
|
||||
return (__m128i) __builtin_ia32_compressdi128_mask ((__v2di) __A,
|
||||
(__v2di) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_maskz_compress_epi64 (__mmask8 __U, __m128i __A) {
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_compress_epi64(__mmask8 __U, __m128i __A) {
|
||||
return (__m128i) __builtin_ia32_compressdi128_mask ((__v2di) __A,
|
||||
(__v2di)
|
||||
_mm_setzero_si128 (),
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_compress_epi64 (__m256i __W, __mmask8 __U, __m256i __A) {
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_compress_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
|
||||
return (__m256i) __builtin_ia32_compressdi256_mask ((__v4di) __A,
|
||||
(__v4di) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_maskz_compress_epi64 (__mmask8 __U, __m256i __A) {
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_maskz_compress_epi64(__mmask8 __U, __m256i __A) {
|
||||
return (__m256i) __builtin_ia32_compressdi256_mask ((__v4di) __A,
|
||||
(__v4di)
|
||||
_mm256_setzero_si256 (),
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_compress_ps (__m128 __W, __mmask8 __U, __m128 __A) {
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_compress_ps(__m128 __W, __mmask8 __U, __m128 __A) {
|
||||
return (__m128) __builtin_ia32_compresssf128_mask ((__v4sf) __A,
|
||||
(__v4sf) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128
|
||||
_mm_maskz_compress_ps (__mmask8 __U, __m128 __A) {
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_compress_ps(__mmask8 __U, __m128 __A) {
|
||||
return (__m128) __builtin_ia32_compresssf128_mask ((__v4sf) __A,
|
||||
(__v4sf)
|
||||
_mm_setzero_ps (),
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256 __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_compress_ps (__m256 __W, __mmask8 __U, __m256 __A) {
|
||||
static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_compress_ps(__m256 __W, __mmask8 __U, __m256 __A) {
|
||||
return (__m256) __builtin_ia32_compresssf256_mask ((__v8sf) __A,
|
||||
(__v8sf) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256 __DEFAULT_FN_ATTRS256
|
||||
_mm256_maskz_compress_ps (__mmask8 __U, __m256 __A) {
|
||||
static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_maskz_compress_ps(__mmask8 __U, __m256 __A) {
|
||||
return (__m256) __builtin_ia32_compresssf256_mask ((__v8sf) __A,
|
||||
(__v8sf)
|
||||
_mm256_setzero_ps (),
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_compress_epi32 (__m128i __W, __mmask8 __U, __m128i __A) {
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_compress_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
|
||||
return (__m128i) __builtin_ia32_compresssi128_mask ((__v4si) __A,
|
||||
(__v4si) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_maskz_compress_epi32 (__mmask8 __U, __m128i __A) {
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_compress_epi32(__mmask8 __U, __m128i __A) {
|
||||
return (__m128i) __builtin_ia32_compresssi128_mask ((__v4si) __A,
|
||||
(__v4si)
|
||||
_mm_setzero_si128 (),
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_compress_epi32 (__m256i __W, __mmask8 __U, __m256i __A) {
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_compress_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
|
||||
return (__m256i) __builtin_ia32_compresssi256_mask ((__v8si) __A,
|
||||
(__v8si) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_maskz_compress_epi32 (__mmask8 __U, __m256i __A) {
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_maskz_compress_epi32(__mmask8 __U, __m256i __A) {
|
||||
return (__m256i) __builtin_ia32_compresssi256_mask ((__v8si) __A,
|
||||
(__v8si)
|
||||
_mm256_setzero_si256 (),
|
||||
@@ -2250,14 +2250,14 @@ _mm256_maskz_div_ps(__mmask8 __U, __m256 __A, __m256 __B) {
|
||||
(__v8sf)_mm256_setzero_ps());
|
||||
}
|
||||
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_expand_pd (__m128d __W, __mmask8 __U, __m128d __A) {
|
||||
return (__m128d) __builtin_ia32_expanddf128_mask ((__v2df) __A,
|
||||
(__v2df) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_expand_pd (__mmask8 __U, __m128d __A) {
|
||||
return (__m128d) __builtin_ia32_expanddf128_mask ((__v2df) __A,
|
||||
(__v2df)
|
||||
@@ -2265,14 +2265,14 @@ _mm_maskz_expand_pd (__mmask8 __U, __m128d __A) {
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256d __DEFAULT_FN_ATTRS256
|
||||
static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_expand_pd (__m256d __W, __mmask8 __U, __m256d __A) {
|
||||
return (__m256d) __builtin_ia32_expanddf256_mask ((__v4df) __A,
|
||||
(__v4df) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256d __DEFAULT_FN_ATTRS256
|
||||
static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_maskz_expand_pd (__mmask8 __U, __m256d __A) {
|
||||
return (__m256d) __builtin_ia32_expanddf256_mask ((__v4df) __A,
|
||||
(__v4df)
|
||||
@@ -2280,14 +2280,14 @@ _mm256_maskz_expand_pd (__mmask8 __U, __m256d __A) {
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_expand_epi64 (__m128i __W, __mmask8 __U, __m128i __A) {
|
||||
return (__m128i) __builtin_ia32_expanddi128_mask ((__v2di) __A,
|
||||
(__v2di) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_expand_epi64 (__mmask8 __U, __m128i __A) {
|
||||
return (__m128i) __builtin_ia32_expanddi128_mask ((__v2di) __A,
|
||||
(__v2di)
|
||||
@@ -2295,14 +2295,14 @@ _mm_maskz_expand_epi64 (__mmask8 __U, __m128i __A) {
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_expand_epi64 (__m256i __W, __mmask8 __U, __m256i __A) {
|
||||
return (__m256i) __builtin_ia32_expanddi256_mask ((__v4di) __A,
|
||||
(__v4di) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_maskz_expand_epi64 (__mmask8 __U, __m256i __A) {
|
||||
return (__m256i) __builtin_ia32_expanddi256_mask ((__v4di) __A,
|
||||
(__v4di)
|
||||
@@ -2445,14 +2445,14 @@ _mm256_maskz_expandloadu_epi32 (__mmask8 __U, void const *__P) {
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_expand_ps (__m128 __W, __mmask8 __U, __m128 __A) {
|
||||
return (__m128) __builtin_ia32_expandsf128_mask ((__v4sf) __A,
|
||||
(__v4sf) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_expand_ps (__mmask8 __U, __m128 __A) {
|
||||
return (__m128) __builtin_ia32_expandsf128_mask ((__v4sf) __A,
|
||||
(__v4sf)
|
||||
@@ -2460,14 +2460,14 @@ _mm_maskz_expand_ps (__mmask8 __U, __m128 __A) {
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256 __DEFAULT_FN_ATTRS256
|
||||
static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_expand_ps (__m256 __W, __mmask8 __U, __m256 __A) {
|
||||
return (__m256) __builtin_ia32_expandsf256_mask ((__v8sf) __A,
|
||||
(__v8sf) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256 __DEFAULT_FN_ATTRS256
|
||||
static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_maskz_expand_ps (__mmask8 __U, __m256 __A) {
|
||||
return (__m256) __builtin_ia32_expandsf256_mask ((__v8sf) __A,
|
||||
(__v8sf)
|
||||
@@ -2475,14 +2475,14 @@ _mm256_maskz_expand_ps (__mmask8 __U, __m256 __A) {
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_expand_epi32 (__m128i __W, __mmask8 __U, __m128i __A) {
|
||||
return (__m128i) __builtin_ia32_expandsi128_mask ((__v4si) __A,
|
||||
(__v4si) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_expand_epi32 (__mmask8 __U, __m128i __A) {
|
||||
return (__m128i) __builtin_ia32_expandsi128_mask ((__v4si) __A,
|
||||
(__v4si)
|
||||
@@ -2490,14 +2490,14 @@ _mm_maskz_expand_epi32 (__mmask8 __U, __m128i __A) {
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_expand_epi32 (__m256i __W, __mmask8 __U, __m256i __A) {
|
||||
return (__m256i) __builtin_ia32_expandsi256_mask ((__v8si) __A,
|
||||
(__v8si) __W,
|
||||
(__mmask8) __U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_maskz_expand_epi32 (__mmask8 __U, __m256i __A) {
|
||||
return (__m256i) __builtin_ia32_expandsi256_mask ((__v8si) __A,
|
||||
(__v8si)
|
||||
@@ -5887,110 +5887,94 @@ _mm256_maskz_permutevar_ps(__mmask8 __U, __m256 __A, __m256i __C) {
|
||||
(__v8sf)_mm256_setzero_ps());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
|
||||
_mm_test_epi32_mask (__m128i __A, __m128i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_test_epi32_mask(__m128i __A, __m128i __B) {
|
||||
return _mm_cmpneq_epi32_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_test_epi32_mask (__mmask8 __U, __m128i __A, __m128i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_test_epi32_mask(__mmask8 __U, __m128i __A, __m128i __B) {
|
||||
return _mm_mask_cmpneq_epi32_mask (__U, _mm_and_si128 (__A, __B),
|
||||
_mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256
|
||||
_mm256_test_epi32_mask (__m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_test_epi32_mask(__m256i __A, __m256i __B) {
|
||||
return _mm256_cmpneq_epi32_mask (_mm256_and_si256 (__A, __B),
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_test_epi32_mask (__mmask8 __U, __m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_test_epi32_mask(__mmask8 __U, __m256i __A, __m256i __B) {
|
||||
return _mm256_mask_cmpneq_epi32_mask (__U, _mm256_and_si256 (__A, __B),
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
|
||||
_mm_test_epi64_mask (__m128i __A, __m128i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_test_epi64_mask(__m128i __A, __m128i __B) {
|
||||
return _mm_cmpneq_epi64_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_test_epi64_mask (__mmask8 __U, __m128i __A, __m128i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_test_epi64_mask(__mmask8 __U, __m128i __A, __m128i __B) {
|
||||
return _mm_mask_cmpneq_epi64_mask (__U, _mm_and_si128 (__A, __B),
|
||||
_mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256
|
||||
_mm256_test_epi64_mask (__m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_test_epi64_mask(__m256i __A, __m256i __B) {
|
||||
return _mm256_cmpneq_epi64_mask (_mm256_and_si256 (__A, __B),
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_test_epi64_mask (__mmask8 __U, __m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_test_epi64_mask(__mmask8 __U, __m256i __A, __m256i __B) {
|
||||
return _mm256_mask_cmpneq_epi64_mask (__U, _mm256_and_si256 (__A, __B),
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
|
||||
_mm_testn_epi32_mask (__m128i __A, __m128i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_testn_epi32_mask(__m128i __A, __m128i __B) {
|
||||
return _mm_cmpeq_epi32_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_testn_epi32_mask (__mmask8 __U, __m128i __A, __m128i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_testn_epi32_mask(__mmask8 __U, __m128i __A, __m128i __B) {
|
||||
return _mm_mask_cmpeq_epi32_mask (__U, _mm_and_si128 (__A, __B),
|
||||
_mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256
|
||||
_mm256_testn_epi32_mask (__m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_testn_epi32_mask(__m256i __A, __m256i __B) {
|
||||
return _mm256_cmpeq_epi32_mask (_mm256_and_si256 (__A, __B),
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_testn_epi32_mask (__mmask8 __U, __m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_testn_epi32_mask(__mmask8 __U, __m256i __A, __m256i __B) {
|
||||
return _mm256_mask_cmpeq_epi32_mask (__U, _mm256_and_si256 (__A, __B),
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
|
||||
_mm_testn_epi64_mask (__m128i __A, __m128i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_testn_epi64_mask(__m128i __A, __m128i __B) {
|
||||
return _mm_cmpeq_epi64_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_testn_epi64_mask (__mmask8 __U, __m128i __A, __m128i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_testn_epi64_mask(__mmask8 __U, __m128i __A, __m128i __B) {
|
||||
return _mm_mask_cmpeq_epi64_mask (__U, _mm_and_si128 (__A, __B),
|
||||
_mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256
|
||||
_mm256_testn_epi64_mask (__m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_testn_epi64_mask(__m256i __A, __m256i __B) {
|
||||
return _mm256_cmpeq_epi64_mask (_mm256_and_si256 (__A, __B),
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_testn_epi64_mask (__mmask8 __U, __m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_testn_epi64_mask(__mmask8 __U, __m256i __A, __m256i __B) {
|
||||
return _mm256_mask_cmpeq_epi64_mask (__U, _mm256_and_si256 (__A, __B),
|
||||
_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
Vendored
+24
-32
@@ -32,33 +32,29 @@
|
||||
#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
|
||||
#endif
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_compress_epi16(__m128i __S, __mmask8 __U, __m128i __D)
|
||||
{
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_compress_epi16(__m128i __S, __mmask8 __U, __m128i __D) {
|
||||
return (__m128i) __builtin_ia32_compresshi128_mask ((__v8hi) __D,
|
||||
(__v8hi) __S,
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_maskz_compress_epi16(__mmask8 __U, __m128i __D)
|
||||
{
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_compress_epi16(__mmask8 __U, __m128i __D) {
|
||||
return (__m128i) __builtin_ia32_compresshi128_mask ((__v8hi) __D,
|
||||
(__v8hi) _mm_setzero_si128(),
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_compress_epi8(__m128i __S, __mmask16 __U, __m128i __D)
|
||||
{
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_compress_epi8(__m128i __S, __mmask16 __U, __m128i __D) {
|
||||
return (__m128i) __builtin_ia32_compressqi128_mask ((__v16qi) __D,
|
||||
(__v16qi) __S,
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_maskz_compress_epi8(__mmask16 __U, __m128i __D)
|
||||
{
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_compress_epi8(__mmask16 __U, __m128i __D) {
|
||||
return (__m128i) __builtin_ia32_compressqi128_mask ((__v16qi) __D,
|
||||
(__v16qi) _mm_setzero_si128(),
|
||||
__U);
|
||||
@@ -78,7 +74,7 @@ _mm_mask_compressstoreu_epi8(void *__P, __mmask16 __U, __m128i __D)
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_expand_epi16(__m128i __S, __mmask8 __U, __m128i __D)
|
||||
{
|
||||
return (__m128i) __builtin_ia32_expandhi128_mask ((__v8hi) __D,
|
||||
@@ -86,7 +82,7 @@ _mm_mask_expand_epi16(__m128i __S, __mmask8 __U, __m128i __D)
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_expand_epi16(__mmask8 __U, __m128i __D)
|
||||
{
|
||||
return (__m128i) __builtin_ia32_expandhi128_mask ((__v8hi) __D,
|
||||
@@ -94,7 +90,7 @@ _mm_maskz_expand_epi16(__mmask8 __U, __m128i __D)
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_mask_expand_epi8(__m128i __S, __mmask16 __U, __m128i __D)
|
||||
{
|
||||
return (__m128i) __builtin_ia32_expandqi128_mask ((__v16qi) __D,
|
||||
@@ -102,7 +98,7 @@ _mm_mask_expand_epi8(__m128i __S, __mmask16 __U, __m128i __D)
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
|
||||
_mm_maskz_expand_epi8(__mmask16 __U, __m128i __D)
|
||||
{
|
||||
return (__m128i) __builtin_ia32_expandqi128_mask ((__v16qi) __D,
|
||||
@@ -142,33 +138,29 @@ _mm_maskz_expandloadu_epi8(__mmask16 __U, void const *__P)
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_compress_epi16(__m256i __S, __mmask16 __U, __m256i __D)
|
||||
{
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_compress_epi16(__m256i __S, __mmask16 __U, __m256i __D) {
|
||||
return (__m256i) __builtin_ia32_compresshi256_mask ((__v16hi) __D,
|
||||
(__v16hi) __S,
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_maskz_compress_epi16(__mmask16 __U, __m256i __D)
|
||||
{
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_maskz_compress_epi16(__mmask16 __U, __m256i __D) {
|
||||
return (__m256i) __builtin_ia32_compresshi256_mask ((__v16hi) __D,
|
||||
(__v16hi) _mm256_setzero_si256(),
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_compress_epi8(__m256i __S, __mmask32 __U, __m256i __D)
|
||||
{
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_compress_epi8(__m256i __S, __mmask32 __U, __m256i __D) {
|
||||
return (__m256i) __builtin_ia32_compressqi256_mask ((__v32qi) __D,
|
||||
(__v32qi) __S,
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_maskz_compress_epi8(__mmask32 __U, __m256i __D)
|
||||
{
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_maskz_compress_epi8(__mmask32 __U, __m256i __D) {
|
||||
return (__m256i) __builtin_ia32_compressqi256_mask ((__v32qi) __D,
|
||||
(__v32qi) _mm256_setzero_si256(),
|
||||
__U);
|
||||
@@ -188,7 +180,7 @@ _mm256_mask_compressstoreu_epi8(void *__P, __mmask32 __U, __m256i __D)
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_expand_epi16(__m256i __S, __mmask16 __U, __m256i __D)
|
||||
{
|
||||
return (__m256i) __builtin_ia32_expandhi256_mask ((__v16hi) __D,
|
||||
@@ -196,7 +188,7 @@ _mm256_mask_expand_epi16(__m256i __S, __mmask16 __U, __m256i __D)
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_maskz_expand_epi16(__mmask16 __U, __m256i __D)
|
||||
{
|
||||
return (__m256i) __builtin_ia32_expandhi256_mask ((__v16hi) __D,
|
||||
@@ -204,7 +196,7 @@ _mm256_maskz_expand_epi16(__mmask16 __U, __m256i __D)
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_mask_expand_epi8(__m256i __S, __mmask32 __U, __m256i __D)
|
||||
{
|
||||
return (__m256i) __builtin_ia32_expandqi256_mask ((__v32qi) __D,
|
||||
@@ -212,7 +204,7 @@ _mm256_mask_expand_epi8(__m256i __S, __mmask32 __U, __m256i __D)
|
||||
__U);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
|
||||
_mm256_maskz_expand_epi8(__mmask32 __U, __m256i __D)
|
||||
{
|
||||
return (__m256i) __builtin_ia32_expandqi256_mask ((__v32qi) __D,
|
||||
|
||||
Vendored
+29
-34
@@ -15,6 +15,16 @@
|
||||
#define __AVX512VLVNNIINTRIN_H
|
||||
|
||||
/* Define the default attributes for the functions in this file. */
|
||||
#if defined(__cplusplus) && (__cplusplus >= 201103L)
|
||||
#define __DEFAULT_FN_ATTRS128 \
|
||||
__attribute__((__always_inline__, __nodebug__, \
|
||||
__target__("avx512vl,avx512vnni"), \
|
||||
__min_vector_width__(128))) constexpr
|
||||
#define __DEFAULT_FN_ATTRS256 \
|
||||
__attribute__((__always_inline__, __nodebug__, \
|
||||
__target__("avx512vl,avx512vnni"), \
|
||||
__min_vector_width__(256))) constexpr
|
||||
#else
|
||||
#define __DEFAULT_FN_ATTRS128 \
|
||||
__attribute__((__always_inline__, __nodebug__, \
|
||||
__target__("avx512vl,avx512vnni"), \
|
||||
@@ -23,6 +33,7 @@
|
||||
__attribute__((__always_inline__, __nodebug__, \
|
||||
__target__("avx512vl,avx512vnni"), \
|
||||
__min_vector_width__(256)))
|
||||
#endif
|
||||
|
||||
/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a A with
|
||||
/// corresponding signed 8-bit integers in \a B, producing 4 intermediate signed
|
||||
@@ -180,128 +191,112 @@
|
||||
((__m128i)__builtin_ia32_vpdpwssds128((__v4si)(S), (__v8hi)(A), (__v8hi)(B)))
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_dpbusd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)
|
||||
{
|
||||
_mm256_mask_dpbusd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
|
||||
return (__m256i)__builtin_ia32_selectd_256(__U,
|
||||
(__v8si)_mm256_dpbusd_epi32(__S, __A, __B),
|
||||
(__v8si)__S);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_maskz_dpbusd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)
|
||||
{
|
||||
_mm256_maskz_dpbusd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) {
|
||||
return (__m256i)__builtin_ia32_selectd_256(__U,
|
||||
(__v8si)_mm256_dpbusd_epi32(__S, __A, __B),
|
||||
(__v8si)_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_dpbusds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)
|
||||
{
|
||||
_mm256_mask_dpbusds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
|
||||
return (__m256i)__builtin_ia32_selectd_256(__U,
|
||||
(__v8si)_mm256_dpbusds_epi32(__S, __A, __B),
|
||||
(__v8si)__S);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_maskz_dpbusds_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpbusds_epi32(
|
||||
__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) {
|
||||
return (__m256i)__builtin_ia32_selectd_256(__U,
|
||||
(__v8si)_mm256_dpbusds_epi32(__S, __A, __B),
|
||||
(__v8si)_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_dpwssd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)
|
||||
{
|
||||
_mm256_mask_dpwssd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
|
||||
return (__m256i)__builtin_ia32_selectd_256(__U,
|
||||
(__v8si)_mm256_dpwssd_epi32(__S, __A, __B),
|
||||
(__v8si)__S);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_maskz_dpwssd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)
|
||||
{
|
||||
_mm256_maskz_dpwssd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) {
|
||||
return (__m256i)__builtin_ia32_selectd_256(__U,
|
||||
(__v8si)_mm256_dpwssd_epi32(__S, __A, __B),
|
||||
(__v8si)_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_mask_dpwssds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)
|
||||
{
|
||||
_mm256_mask_dpwssds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
|
||||
return (__m256i)__builtin_ia32_selectd_256(__U,
|
||||
(__v8si)_mm256_dpwssds_epi32(__S, __A, __B),
|
||||
(__v8si)__S);
|
||||
}
|
||||
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_maskz_dpwssds_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B)
|
||||
{
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpwssds_epi32(
|
||||
__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) {
|
||||
return (__m256i)__builtin_ia32_selectd_256(__U,
|
||||
(__v8si)_mm256_dpwssds_epi32(__S, __A, __B),
|
||||
(__v8si)_mm256_setzero_si256());
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_dpbusd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)
|
||||
{
|
||||
_mm_mask_dpbusd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
|
||||
return (__m128i)__builtin_ia32_selectd_128(__U,
|
||||
(__v4si)_mm_dpbusd_epi32(__S, __A, __B),
|
||||
(__v4si)__S);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_maskz_dpbusd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B)
|
||||
{
|
||||
_mm_maskz_dpbusd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) {
|
||||
return (__m128i)__builtin_ia32_selectd_128(__U,
|
||||
(__v4si)_mm_dpbusd_epi32(__S, __A, __B),
|
||||
(__v4si)_mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_dpbusds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)
|
||||
{
|
||||
_mm_mask_dpbusds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
|
||||
return (__m128i)__builtin_ia32_selectd_128(__U,
|
||||
(__v4si)_mm_dpbusds_epi32(__S, __A, __B),
|
||||
(__v4si)__S);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_maskz_dpbusds_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B)
|
||||
{
|
||||
_mm_maskz_dpbusds_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) {
|
||||
return (__m128i)__builtin_ia32_selectd_128(__U,
|
||||
(__v4si)_mm_dpbusds_epi32(__S, __A, __B),
|
||||
(__v4si)_mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_dpwssd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)
|
||||
{
|
||||
_mm_mask_dpwssd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
|
||||
return (__m128i)__builtin_ia32_selectd_128(__U,
|
||||
(__v4si)_mm_dpwssd_epi32(__S, __A, __B),
|
||||
(__v4si)__S);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_maskz_dpwssd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B)
|
||||
{
|
||||
_mm_maskz_dpwssd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) {
|
||||
return (__m128i)__builtin_ia32_selectd_128(__U,
|
||||
(__v4si)_mm_dpwssd_epi32(__S, __A, __B),
|
||||
(__v4si)_mm_setzero_si128());
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_mask_dpwssds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B)
|
||||
{
|
||||
_mm_mask_dpwssds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
|
||||
return (__m128i)__builtin_ia32_selectd_128(__U,
|
||||
(__v4si)_mm_dpwssds_epi32(__S, __A, __B),
|
||||
(__v4si)__S);
|
||||
}
|
||||
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_maskz_dpwssds_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B)
|
||||
{
|
||||
_mm_maskz_dpwssds_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) {
|
||||
return (__m128i)__builtin_ia32_selectd_128(__U,
|
||||
(__v4si)_mm_dpwssds_epi32(__S, __A, __B),
|
||||
(__v4si)_mm_setzero_si128());
|
||||
|
||||
Vendored
+32
-35
@@ -15,102 +15,99 @@
|
||||
#define __AVX512VNNIINTRIN_H
|
||||
|
||||
/* Define the default attributes for the functions in this file. */
|
||||
#if defined(__cplusplus) && (__cplusplus >= 201103L)
|
||||
#define __DEFAULT_FN_ATTRS \
|
||||
__attribute__((__always_inline__, __nodebug__, __target__("avx512vnni"), \
|
||||
__min_vector_width__(512))) constexpr
|
||||
#else
|
||||
#define __DEFAULT_FN_ATTRS \
|
||||
__attribute__((__always_inline__, __nodebug__, __target__("avx512vnni"), \
|
||||
__min_vector_width__(512)))
|
||||
#endif
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_dpbusd_epi32(__m512i __S, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbusd_epi32(__m512i __S,
|
||||
__m512i __A,
|
||||
__m512i __B) {
|
||||
return (__m512i)__builtin_ia32_vpdpbusd512((__v16si)__S, (__v64qu)__A,
|
||||
(__v64qi)__B);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_mask_dpbusd_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B)
|
||||
{
|
||||
_mm512_mask_dpbusd_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) {
|
||||
return (__m512i)__builtin_ia32_selectd_512(__U,
|
||||
(__v16si)_mm512_dpbusd_epi32(__S, __A, __B),
|
||||
(__v16si)__S);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_maskz_dpbusd_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbusd_epi32(
|
||||
__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) {
|
||||
return (__m512i)__builtin_ia32_selectd_512(__U,
|
||||
(__v16si)_mm512_dpbusd_epi32(__S, __A, __B),
|
||||
(__v16si)_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_dpbusds_epi32(__m512i __S, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbusds_epi32(__m512i __S,
|
||||
__m512i __A,
|
||||
__m512i __B) {
|
||||
return (__m512i)__builtin_ia32_vpdpbusds512((__v16si)__S, (__v64qu)__A,
|
||||
(__v64qi)__B);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_mask_dpbusds_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpbusds_epi32(
|
||||
__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) {
|
||||
return (__m512i)__builtin_ia32_selectd_512(__U,
|
||||
(__v16si)_mm512_dpbusds_epi32(__S, __A, __B),
|
||||
(__v16si)__S);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_maskz_dpbusds_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbusds_epi32(
|
||||
__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) {
|
||||
return (__m512i)__builtin_ia32_selectd_512(__U,
|
||||
(__v16si)_mm512_dpbusds_epi32(__S, __A, __B),
|
||||
(__v16si)_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_dpwssd_epi32(__m512i __S, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwssd_epi32(__m512i __S,
|
||||
__m512i __A,
|
||||
__m512i __B) {
|
||||
return (__m512i)__builtin_ia32_vpdpwssd512((__v16si)__S, (__v32hi)__A,
|
||||
(__v32hi)__B);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_mask_dpwssd_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B)
|
||||
{
|
||||
_mm512_mask_dpwssd_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) {
|
||||
return (__m512i)__builtin_ia32_selectd_512(__U,
|
||||
(__v16si)_mm512_dpwssd_epi32(__S, __A, __B),
|
||||
(__v16si)__S);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_maskz_dpwssd_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwssd_epi32(
|
||||
__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) {
|
||||
return (__m512i)__builtin_ia32_selectd_512(__U,
|
||||
(__v16si)_mm512_dpwssd_epi32(__S, __A, __B),
|
||||
(__v16si)_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_dpwssds_epi32(__m512i __S, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwssds_epi32(__m512i __S,
|
||||
__m512i __A,
|
||||
__m512i __B) {
|
||||
return (__m512i)__builtin_ia32_vpdpwssds512((__v16si)__S, (__v32hi)__A,
|
||||
(__v32hi)__B);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_mask_dpwssds_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwssds_epi32(
|
||||
__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) {
|
||||
return (__m512i)__builtin_ia32_selectd_512(__U,
|
||||
(__v16si)_mm512_dpwssds_epi32(__S, __A, __B),
|
||||
(__v16si)__S);
|
||||
}
|
||||
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS
|
||||
_mm512_maskz_dpwssds_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B)
|
||||
{
|
||||
static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwssds_epi32(
|
||||
__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) {
|
||||
return (__m512i)__builtin_ia32_selectd_512(__U,
|
||||
(__v16si)_mm512_dpwssds_epi32(__S, __A, __B),
|
||||
(__v16si)_mm512_setzero_si512());
|
||||
}
|
||||
|
||||
#undef __DEFAULT_FN_ATTRS
|
||||
|
||||
#endif
|
||||
+1
-1
@@ -22,7 +22,7 @@
|
||||
*===-----------------------------------------------------------------------===
|
||||
*/
|
||||
#ifndef __IMMINTRIN_H
|
||||
#error "Never use <avx512vp2intersect.h> directly; include <immintrin.h> instead."
|
||||
#error "Never use <avx512vp2intersectintrin.h> directly; include <immintrin.h> instead."
|
||||
#endif
|
||||
|
||||
#ifndef _AVX512VP2INTERSECT_H
|
||||
|
||||
Vendored
+23
-18
@@ -40,8 +40,21 @@
|
||||
|
||||
/* Intrinsics with _avx_ prefix are for compatibility with msvc. */
|
||||
/* Define the default attributes for the functions in this file. */
|
||||
#define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), __min_vector_width__(256)))
|
||||
#define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), __min_vector_width__(128)))
|
||||
#if defined(__cplusplus) && (__cplusplus >= 201103L)
|
||||
#define __DEFAULT_FN_ATTRS256 \
|
||||
__attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), \
|
||||
__min_vector_width__(256))) constexpr
|
||||
#define __DEFAULT_FN_ATTRS128 \
|
||||
__attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), \
|
||||
__min_vector_width__(128))) constexpr
|
||||
#else
|
||||
#define __DEFAULT_FN_ATTRS256 \
|
||||
__attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), \
|
||||
__min_vector_width__(256)))
|
||||
#define __DEFAULT_FN_ATTRS128 \
|
||||
__attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), \
|
||||
__min_vector_width__(128)))
|
||||
#endif
|
||||
|
||||
/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
|
||||
/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate signed
|
||||
@@ -61,8 +74,7 @@
|
||||
/// DST[MAX:256] := 0
|
||||
/// \endcode
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_dpbusd_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
|
||||
{
|
||||
_mm256_dpbusd_avx_epi32(__m256i __S, __m256i __A, __m256i __B) {
|
||||
return (__m256i)__builtin_ia32_vpdpbusd256((__v8si)__S, (__v32qu)__A,
|
||||
(__v32qi)__B);
|
||||
}
|
||||
@@ -85,8 +97,7 @@ _mm256_dpbusd_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
|
||||
/// DST[MAX:256] := 0
|
||||
/// \endcode
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_dpbusds_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
|
||||
{
|
||||
_mm256_dpbusds_avx_epi32(__m256i __S, __m256i __A, __m256i __B) {
|
||||
return (__m256i)__builtin_ia32_vpdpbusds256((__v8si)__S, (__v32qu)__A,
|
||||
(__v32qi)__B);
|
||||
}
|
||||
@@ -107,8 +118,7 @@ _mm256_dpbusds_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
|
||||
/// DST[MAX:256] := 0
|
||||
/// \endcode
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_dpwssd_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
|
||||
{
|
||||
_mm256_dpwssd_avx_epi32(__m256i __S, __m256i __A, __m256i __B) {
|
||||
return (__m256i)__builtin_ia32_vpdpwssd256((__v8si)__S, (__v16hi)__A,
|
||||
(__v16hi)__B);
|
||||
}
|
||||
@@ -129,8 +139,7 @@ _mm256_dpwssd_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
|
||||
/// DST[MAX:256] := 0
|
||||
/// \endcode
|
||||
static __inline__ __m256i __DEFAULT_FN_ATTRS256
|
||||
_mm256_dpwssds_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
|
||||
{
|
||||
_mm256_dpwssds_avx_epi32(__m256i __S, __m256i __A, __m256i __B) {
|
||||
return (__m256i)__builtin_ia32_vpdpwssds256((__v8si)__S, (__v16hi)__A,
|
||||
(__v16hi)__B);
|
||||
}
|
||||
@@ -153,8 +162,7 @@ _mm256_dpwssds_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
|
||||
/// DST[MAX:128] := 0
|
||||
/// \endcode
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_dpbusd_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
|
||||
{
|
||||
_mm_dpbusd_avx_epi32(__m128i __S, __m128i __A, __m128i __B) {
|
||||
return (__m128i)__builtin_ia32_vpdpbusd128((__v4si)__S, (__v16qu)__A,
|
||||
(__v16qi)__B);
|
||||
}
|
||||
@@ -177,8 +185,7 @@ _mm_dpbusd_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
|
||||
/// DST[MAX:128] := 0
|
||||
/// \endcode
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_dpbusds_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
|
||||
{
|
||||
_mm_dpbusds_avx_epi32(__m128i __S, __m128i __A, __m128i __B) {
|
||||
return (__m128i)__builtin_ia32_vpdpbusds128((__v4si)__S, (__v16qu)__A,
|
||||
(__v16qi)__B);
|
||||
}
|
||||
@@ -199,8 +206,7 @@ _mm_dpbusds_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
|
||||
/// DST[MAX:128] := 0
|
||||
/// \endcode
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_dpwssd_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
|
||||
{
|
||||
_mm_dpwssd_avx_epi32(__m128i __S, __m128i __A, __m128i __B) {
|
||||
return (__m128i)__builtin_ia32_vpdpwssd128((__v4si)__S, (__v8hi)__A,
|
||||
(__v8hi)__B);
|
||||
}
|
||||
@@ -221,8 +227,7 @@ _mm_dpwssd_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
|
||||
/// DST[MAX:128] := 0
|
||||
/// \endcode
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS128
|
||||
_mm_dpwssds_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
|
||||
{
|
||||
_mm_dpwssds_avx_epi32(__m128i __S, __m128i __A, __m128i __B) {
|
||||
return (__m128i)__builtin_ia32_vpdpwssds128((__v4si)__S, (__v8hi)__A,
|
||||
(__v8hi)__B);
|
||||
}
|
||||
|
||||
Vendored
+10
-8
@@ -10,8 +10,14 @@
|
||||
#ifndef __CRC32INTRIN_H
|
||||
#define __CRC32INTRIN_H
|
||||
|
||||
/// We only declare crc32 as a constexpr if we are compiling C++ code
|
||||
#if defined(__cplusplus) && (__cplusplus >= 201103L)
|
||||
#define __DEFAULT_FN_ATTRS \
|
||||
__attribute__((__always_inline__, __nodebug__, __target__("crc32"))) constexpr
|
||||
#else
|
||||
#define __DEFAULT_FN_ATTRS \
|
||||
__attribute__((__always_inline__, __nodebug__, __target__("crc32")))
|
||||
#endif
|
||||
|
||||
/// Adds the unsigned integer operand to the CRC-32C checksum of the
|
||||
/// unsigned char operand.
|
||||
@@ -28,8 +34,7 @@
|
||||
/// \returns The result of adding operand \a __C to the CRC-32C checksum of
|
||||
/// operand \a __D.
|
||||
static __inline__ unsigned int __DEFAULT_FN_ATTRS
|
||||
_mm_crc32_u8(unsigned int __C, unsigned char __D)
|
||||
{
|
||||
_mm_crc32_u8(unsigned int __C, unsigned char __D) {
|
||||
return __builtin_ia32_crc32qi(__C, __D);
|
||||
}
|
||||
|
||||
@@ -48,8 +53,7 @@ _mm_crc32_u8(unsigned int __C, unsigned char __D)
|
||||
/// \returns The result of adding operand \a __C to the CRC-32C checksum of
|
||||
/// operand \a __D.
|
||||
static __inline__ unsigned int __DEFAULT_FN_ATTRS
|
||||
_mm_crc32_u16(unsigned int __C, unsigned short __D)
|
||||
{
|
||||
_mm_crc32_u16(unsigned int __C, unsigned short __D) {
|
||||
return __builtin_ia32_crc32hi(__C, __D);
|
||||
}
|
||||
|
||||
@@ -68,8 +72,7 @@ _mm_crc32_u16(unsigned int __C, unsigned short __D)
|
||||
/// \returns The result of adding operand \a __C to the CRC-32C checksum of
|
||||
/// operand \a __D.
|
||||
static __inline__ unsigned int __DEFAULT_FN_ATTRS
|
||||
_mm_crc32_u32(unsigned int __C, unsigned int __D)
|
||||
{
|
||||
_mm_crc32_u32(unsigned int __C, unsigned int __D) {
|
||||
return __builtin_ia32_crc32si(__C, __D);
|
||||
}
|
||||
|
||||
@@ -89,8 +92,7 @@ _mm_crc32_u32(unsigned int __C, unsigned int __D)
|
||||
/// \returns The result of adding operand \a __C to the CRC-32C checksum of
|
||||
/// operand \a __D.
|
||||
static __inline__ unsigned long long __DEFAULT_FN_ATTRS
|
||||
_mm_crc32_u64(unsigned long long __C, unsigned long long __D)
|
||||
{
|
||||
_mm_crc32_u64(unsigned long long __C, unsigned long long __D) {
|
||||
return __builtin_ia32_crc32di(__C, __D);
|
||||
}
|
||||
#endif /* __x86_64__ */
|
||||
|
||||
Vendored
+8
-8
@@ -279,8 +279,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sqrt_pd(__m128d __a) {
|
||||
/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the
|
||||
/// minimum value between both operands. The upper 64 bits are copied from
|
||||
/// the upper 64 bits of the first source operand.
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_min_sd(__m128d __a,
|
||||
__m128d __b) {
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_min_sd(__m128d __a,
|
||||
__m128d __b) {
|
||||
return __builtin_ia32_minsd((__v2df)__a, (__v2df)__b);
|
||||
}
|
||||
|
||||
@@ -325,8 +325,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_min_pd(__m128d __a,
|
||||
/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the
|
||||
/// maximum value between both operands. The upper 64 bits are copied from
|
||||
/// the upper 64 bits of the first source operand.
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_max_sd(__m128d __a,
|
||||
__m128d __b) {
|
||||
static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_max_sd(__m128d __a,
|
||||
__m128d __b) {
|
||||
return __builtin_ia32_maxsd((__v2df)__a, (__v2df)__b);
|
||||
}
|
||||
|
||||
@@ -1343,7 +1343,7 @@ _mm_cvtepi32_pd(__m128i __a) {
|
||||
/// \returns A 128-bit vector of [4 x i32] whose lower 64 bits contain the
|
||||
/// converted values. The upper 64 bits are set to zero.
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtpd_epi32(__m128d __a) {
|
||||
return __builtin_ia32_cvtpd2dq((__v2df)__a);
|
||||
return (__m128i)__builtin_ia32_cvtpd2dq((__v2df)__a);
|
||||
}
|
||||
|
||||
/// Converts the low-order element of a 128-bit vector of [2 x double]
|
||||
@@ -2481,9 +2481,9 @@ _mm_mul_epu32(__m128i __a, __m128i __b) {
|
||||
/// A 128-bit integer vector containing one of the source operands.
|
||||
/// \returns A [2 x i64] vector containing the sums of the sets of absolute
|
||||
/// differences between both operands.
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sad_epu8(__m128i __a,
|
||||
__m128i __b) {
|
||||
return __builtin_ia32_psadbw128((__v16qi)__a, (__v16qi)__b);
|
||||
static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
|
||||
_mm_sad_epu8(__m128i __a, __m128i __b) {
|
||||
return __builtin_ia32_psadbw128((__v16qu)__a, (__v16qu)__b);
|
||||
}
|
||||
|
||||
/// Subtracts the corresponding 8-bit integer values in the operands.
|
||||
|
||||
Vendored
+91
@@ -0,0 +1,91 @@
|
||||
//===-- Definition of macros from endian.h --------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CLANG_ENDIAN_H
|
||||
#define __CLANG_ENDIAN_H
|
||||
|
||||
// If the system has an endian.h, let's use that instead.
|
||||
#if __has_include_next(<endian.h>)
|
||||
#include_next <endian.h>
|
||||
#else
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
// Implementation taken from llvm libc endian-macros.h.
|
||||
#ifdef __cplusplus
|
||||
#define __CLANG_ENDIAN_CAST(cast, type, value) (cast<type>(value))
|
||||
#else
|
||||
#define __CLANG_ENDIAN_CAST(cast, type, value) ((type)(value))
|
||||
#endif
|
||||
|
||||
#define LITTLE_ENDIAN __ORDER_LITTLE_ENDIAN__
|
||||
#define BIG_ENDIAN __ORDER_BIG_ENDIAN__
|
||||
#define PDP_ENDIAN __ORDER_PDP_ENDIAN__
|
||||
#define BYTE_ORDER __BYTE_ORDER__
|
||||
|
||||
// Define some compatibility macros if they are not defined.
|
||||
#ifndef __BYTE_ORDER
|
||||
#define __BYTE_ORDER BYTE_ORDER
|
||||
#endif
|
||||
#ifndef __LITTLE_ENDIAN
|
||||
#define __LITTLE_ENDIAN LITTLE_ENDIAN
|
||||
#endif
|
||||
#ifndef __BIG_ENDIAN
|
||||
#define __BIG_ENDIAN BIG_ENDIAN
|
||||
#endif
|
||||
#ifndef __PDP_ENDIAN
|
||||
#define __PDP_ENDIAN PDP_ENDIAN
|
||||
#endif
|
||||
|
||||
#if BYTE_ORDER == LITTLE_ENDIAN
|
||||
|
||||
#define htobe16(x) \
|
||||
__builtin_bswap16(__CLANG_ENDIAN_CAST(static_cast, uint16_t, x))
|
||||
#define htobe32(x) \
|
||||
__builtin_bswap32(__CLANG_ENDIAN_CAST(static_cast, uint32_t, x))
|
||||
#define htobe64(x) \
|
||||
__builtin_bswap64(__CLANG_ENDIAN_CAST(static_cast, uint64_t, x))
|
||||
#define htole16(x) __CLANG_ENDIAN_CAST(static_cast, uint16_t, x)
|
||||
#define htole32(x) __CLANG_ENDIAN_CAST(static_cast, uint32_t, x)
|
||||
#define htole64(x) __CLANG_ENDIAN_CAST(static_cast, uint64_t, x)
|
||||
#define be16toh(x) \
|
||||
__builtin_bswap16(__CLANG_ENDIAN_CAST(static_cast, uint16_t, x))
|
||||
#define be32toh(x) \
|
||||
__builtin_bswap32(__CLANG_ENDIAN_CAST(static_cast, uint32_t, x))
|
||||
#define be64toh(x) \
|
||||
__builtin_bswap64(__CLANG_ENDIAN_CAST(static_cast, uint64_t, x))
|
||||
#define le16toh(x) __CLANG_ENDIAN_CAST(static_cast, uint16_t, x)
|
||||
#define le32toh(x) __CLANG_ENDIAN_CAST(static_cast, uint32_t, x)
|
||||
#define le64toh(x) __CLANG_ENDIAN_CAST(static_cast, uint64_t, x)
|
||||
|
||||
#elif BYTE_ORDER == BIG_ENDIAN
|
||||
|
||||
#define htobe16(x) __CLANG_ENDIAN_CAST(static_cast, uint16_t, x)
|
||||
#define htobe32(x) __CLANG_ENDIAN_CAST(static_cast, uint32_t, x)
|
||||
#define htobe64(x) __CLANG_ENDIAN_CAST(static_cast, uint64_t, x)
|
||||
#define htole16(x) \
|
||||
__builtin_bswap16(__CLANG_ENDIAN_CAST(static_cast, uint16_t, x))
|
||||
#define htole32(x) \
|
||||
__builtin_bswap32(__CLANG_ENDIAN_CAST(static_cast, uint32_t, x))
|
||||
#define htole64(x) \
|
||||
__builtin_bswap64(__CLANG_ENDIAN_CAST(static_cast, uint64_t, x))
|
||||
#define be16toh(x) __CLANG_ENDIAN_CAST(static_cast, uint16_t, x)
|
||||
#define be32toh(x) __CLANG_ENDIAN_CAST(static_cast, uint32_t, x)
|
||||
#define be64toh(x) __CLANG_ENDIAN_CAST(static_cast, uint64_t, x)
|
||||
#define le16toh(x) \
|
||||
__builtin_bswap16(__CLANG_ENDIAN_CAST(static_cast, uint16_t, x))
|
||||
#define le32toh(x) \
|
||||
__builtin_bswap32(__CLANG_ENDIAN_CAST(static_cast, uint32_t, x))
|
||||
#define le64toh(x) \
|
||||
__builtin_bswap64(__CLANG_ENDIAN_CAST(static_cast, uint64_t, x))
|
||||
|
||||
#else
|
||||
#error "Unsupported endianness"
|
||||
#endif
|
||||
#endif // __has_include_next
|
||||
#endif // __CLANG_ENDIAN_H
|
||||
Vendored
+102
-80
@@ -4577,8 +4577,9 @@
|
||||
#if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vdd32.sf=vadd(Vu32.bf,Vv32.bf)
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vadd_VbfVbf(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vadd_VbfVbf(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VX_DV
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Wsf_vadd_VbfVbf(Vu, Vv) \
|
||||
@@ -4636,8 +4637,9 @@
|
||||
#if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vd32.bf=vcvt(Vu32.sf,Vv32.sf)
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vbf_vcvt_VsfVsf(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VX Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vbf_vcvt_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VX
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Vbf_vcvt_VsfVsf(Vu, Vv) \
|
||||
@@ -4647,8 +4649,9 @@
|
||||
#if __HVX_ARCH__ >= 73
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Qd4=vcmp.gt(Vu32.bf,Vv32.bf)
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VbfVbf(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VA Execution Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VbfVbf(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VA
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Q_vcmp_gt_VbfVbf(Vu, Vv) \
|
||||
@@ -4659,9 +4662,9 @@
|
||||
#if __HVX_ARCH__ >= 73
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Qx4&=vcmp.gt(Vu32.bf,Vv32.bf)
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVbfVbf(HVX_VectorPred
|
||||
Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
|
||||
Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVbfVbf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VA
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Q_vcmp_gtand_QVbfVbf(Qx, Vu, Vv) \
|
||||
@@ -4675,9 +4678,9 @@
|
||||
#if __HVX_ARCH__ >= 73
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Qx4|=vcmp.gt(Vu32.bf,Vv32.bf)
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVbfVbf(HVX_VectorPred
|
||||
Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
|
||||
Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVbfVbf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VA
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Q_vcmp_gtor_QVbfVbf(Qx, Vu, Vv) \
|
||||
@@ -4691,9 +4694,9 @@
|
||||
#if __HVX_ARCH__ >= 73
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Qx4^=vcmp.gt(Vu32.bf,Vv32.bf)
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVbfVbf(HVX_VectorPred
|
||||
Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
|
||||
Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVbfVbf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VA
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Q_vcmp_gtxacc_QVbfVbf(Qx, Vu, Vv) \
|
||||
@@ -4707,8 +4710,9 @@
|
||||
#if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vd32.bf=vmax(Vu32.bf,Vv32.bf)
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vbf_vmax_VbfVbf(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VX_LATE Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vbf_vmax_VbfVbf(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VX_LATE
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Vbf_vmax_VbfVbf(Vu, Vv) \
|
||||
@@ -4718,8 +4722,9 @@
|
||||
#if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vd32.bf=vmin(Vu32.bf,Vv32.bf)
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vbf_vmin_VbfVbf(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VX_LATE Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vbf_vmin_VbfVbf(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VX_LATE
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Vbf_vmin_VbfVbf(Vu, Vv) \
|
||||
@@ -4729,8 +4734,9 @@
|
||||
#if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vdd32.sf=vmpy(Vu32.bf,Vv32.bf)
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpy_VbfVbf(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpy_VbfVbf(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VX_DV
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Wsf_vmpy_VbfVbf(Vu, Vv) \
|
||||
@@ -4740,9 +4746,9 @@
|
||||
#if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vxx32.sf+=vmpy(Vu32.bf,Vv32.bf)
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpyacc_WsfVbfVbf(HVX_VectorPair
|
||||
Vxx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution
|
||||
Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpyacc_WsfVbfVbf(HVX_VectorPair Vxx, HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VX_DV
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Wsf_vmpyacc_WsfVbfVbf(Vxx, Vu, Vv) \
|
||||
@@ -4752,8 +4758,9 @@
|
||||
#if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vdd32.sf=vsub(Vu32.bf,Vv32.bf)
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vsub_VbfVbf(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vsub_VbfVbf(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VX_DV
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Wsf_vsub_VbfVbf(Vu, Vv) \
|
||||
@@ -4775,9 +4782,9 @@
|
||||
#if __HVX_ARCH__ >= 79
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vx32|=vgetqfext(Vu32.x,Rt32)
|
||||
C Intrinsic Prototype: HVX_Vector Q6_V_vgetqfextor_VVR(HVX_Vector Vx,
|
||||
HVX_Vector Vu, Word32 Rt) Instruction Type: CVI_VX Execution Slots:
|
||||
SLOT23
|
||||
C Intrinsic Prototype: HVX_Vector Q6_V_vgetqfextor_VVR(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt)
|
||||
Instruction Type: CVI_VX
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_V_vgetqfextor_VVR(Vx, Vu, Rt) \
|
||||
@@ -4810,8 +4817,9 @@
|
||||
#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vdd32.hf=vadd(Vu32.f8,Vv32.f8)
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vadd_VV(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vadd_VV(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VX_DV
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Whf_vadd_VV(Vu, Vv) \
|
||||
@@ -4821,8 +4829,9 @@
|
||||
#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vd32.b=vcvt2(Vu32.hf,Vv32.hf)
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vb_vcvt2_VhfVhf(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VX Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vb_vcvt2_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VX
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Vb_vcvt2_VhfVhf(Vu, Vv) \
|
||||
@@ -4856,8 +4865,9 @@
|
||||
#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vd32.ub=vcvt2(Vu32.hf,Vv32.hf)
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vub_vcvt2_VhfVhf(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VX Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vub_vcvt2_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VX
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Vub_vcvt2_VhfVhf(Vu, Vv) \
|
||||
@@ -4867,8 +4877,9 @@
|
||||
#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vd32.f8=vcvt(Vu32.hf,Vv32.hf)
|
||||
C Intrinsic Prototype: HVX_Vector Q6_V_vcvt_VhfVhf(HVX_Vector Vu, HVX_Vector
|
||||
Vv) Instruction Type: CVI_VX Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_Vector Q6_V_vcvt_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VX
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_V_vcvt_VhfVhf(Vu, Vv) \
|
||||
@@ -4926,8 +4937,9 @@
|
||||
#if __HVX_ARCH__ >= 79
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vd32=vmerge(Vu32.x,Vv32.w)
|
||||
C Intrinsic Prototype: HVX_Vector Q6_V_vmerge_VVw(HVX_Vector Vu, HVX_Vector
|
||||
Vv) Instruction Type: CVI_VS Execution Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_Vector Q6_V_vmerge_VVw(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VS
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_V_vmerge_VVw(Vu, Vv) \
|
||||
@@ -4937,8 +4949,9 @@
|
||||
#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vdd32.hf=vmpy(Vu32.f8,Vv32.f8)
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vmpy_VV(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vmpy_VV(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VX_DV
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Whf_vmpy_VV(Vu, Vv) \
|
||||
@@ -4948,9 +4961,9 @@
|
||||
#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vxx32.hf+=vmpy(Vu32.f8,Vv32.f8)
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vmpyacc_WhfVV(HVX_VectorPair
|
||||
Vxx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution
|
||||
Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vmpyacc_WhfVV(HVX_VectorPair Vxx, HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VX_DV
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Whf_vmpyacc_WhfVV(Vxx, Vu, Vv) \
|
||||
@@ -4960,8 +4973,9 @@
|
||||
#if __HVX_ARCH__ >= 79
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vd32.qf16=vmpy(Vu32.hf,Rt32.hf)
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_VhfRhf(HVX_Vector Vu, Word32
|
||||
Rt) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_VhfRhf(HVX_Vector Vu, Word32 Rt)
|
||||
Instruction Type: CVI_VX_DV
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Vqf16_vmpy_VhfRhf(Vu, Rt) \
|
||||
@@ -4971,8 +4985,9 @@
|
||||
#if __HVX_ARCH__ >= 79
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vd32.qf16=vmpy(Vu32.qf16,Rt32.hf)
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_Vqf16Rhf(HVX_Vector Vu,
|
||||
Word32 Rt) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_Vqf16Rhf(HVX_Vector Vu, Word32 Rt)
|
||||
Instruction Type: CVI_VX_DV
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Vqf16_vmpy_Vqf16Rhf(Vu, Rt) \
|
||||
@@ -4982,8 +4997,9 @@
|
||||
#if __HVX_ARCH__ >= 79
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vd32.qf32=vmpy(Vu32.sf,Rt32.sf)
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vmpy_VsfRsf(HVX_Vector Vu, Word32
|
||||
Rt) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vmpy_VsfRsf(HVX_Vector Vu, Word32 Rt)
|
||||
Instruction Type: CVI_VX_DV
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Vqf32_vmpy_VsfRsf(Vu, Rt) \
|
||||
@@ -4993,8 +5009,9 @@
|
||||
#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vdd32.hf=vsub(Vu32.f8,Vv32.f8)
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vsub_VV(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
|
||||
C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vsub_VV(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VX_DV
|
||||
Execution Slots: SLOT23
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Whf_vsub_VV(Vu, Vv) \
|
||||
@@ -5052,8 +5069,9 @@
|
||||
#if __HVX_ARCH__ >= 81
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vd32=valign4(Vu32,Vv32,Rt8)
|
||||
C Intrinsic Prototype: HVX_Vector Q6_V_valign4_VVR(HVX_Vector Vu, HVX_Vector
|
||||
Vv, Word32 Rt) Instruction Type: CVI_VA Execution Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_Vector Q6_V_valign4_VVR(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
|
||||
Instruction Type: CVI_VA
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_V_valign4_VVR(Vu, Vv, Rt) \
|
||||
@@ -5159,8 +5177,9 @@
|
||||
#if __HVX_ARCH__ >= 81
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Qd4=vcmp.eq(Vu32.hf,Vv32.hf)
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eq_VhfVhf(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VA Execution Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eq_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VA
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Q_vcmp_eq_VhfVhf(Vu, Vv) \
|
||||
@@ -5171,9 +5190,9 @@
|
||||
#if __HVX_ARCH__ >= 81
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Qx4&=vcmp.eq(Vu32.hf,Vv32.hf)
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqand_QVhfVhf(HVX_VectorPred
|
||||
Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
|
||||
Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqand_QVhfVhf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VA
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Q_vcmp_eqand_QVhfVhf(Qx, Vu, Vv) \
|
||||
@@ -5187,9 +5206,9 @@
|
||||
#if __HVX_ARCH__ >= 81
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Qx4|=vcmp.eq(Vu32.hf,Vv32.hf)
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqor_QVhfVhf(HVX_VectorPred
|
||||
Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
|
||||
Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqor_QVhfVhf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VA
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Q_vcmp_eqor_QVhfVhf(Qx, Vu, Vv) \
|
||||
@@ -5203,9 +5222,9 @@
|
||||
#if __HVX_ARCH__ >= 81
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Qx4^=vcmp.eq(Vu32.hf,Vv32.hf)
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqxacc_QVhfVhf(HVX_VectorPred
|
||||
Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
|
||||
Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqxacc_QVhfVhf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VA
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Q_vcmp_eqxacc_QVhfVhf(Qx, Vu, Vv) \
|
||||
@@ -5219,8 +5238,9 @@
|
||||
#if __HVX_ARCH__ >= 81
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Qd4=vcmp.eq(Vu32.sf,Vv32.sf)
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eq_VsfVsf(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VA Execution Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eq_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VA
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Q_vcmp_eq_VsfVsf(Vu, Vv) \
|
||||
@@ -5231,9 +5251,9 @@
|
||||
#if __HVX_ARCH__ >= 81
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Qx4&=vcmp.eq(Vu32.sf,Vv32.sf)
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqand_QVsfVsf(HVX_VectorPred
|
||||
Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
|
||||
Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqand_QVsfVsf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VA
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Q_vcmp_eqand_QVsfVsf(Qx, Vu, Vv) \
|
||||
@@ -5247,9 +5267,9 @@
|
||||
#if __HVX_ARCH__ >= 81
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Qx4|=vcmp.eq(Vu32.sf,Vv32.sf)
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqor_QVsfVsf(HVX_VectorPred
|
||||
Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
|
||||
Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqor_QVsfVsf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VA
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Q_vcmp_eqor_QVsfVsf(Qx, Vu, Vv) \
|
||||
@@ -5263,9 +5283,9 @@
|
||||
#if __HVX_ARCH__ >= 81
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Qx4^=vcmp.eq(Vu32.sf,Vv32.sf)
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqxacc_QVsfVsf(HVX_VectorPred
|
||||
Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
|
||||
Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqxacc_QVsfVsf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VA
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Q_vcmp_eqxacc_QVsfVsf(Qx, Vu, Vv) \
|
||||
@@ -5375,8 +5395,9 @@
|
||||
#if __HVX_ARCH__ >= 81
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vd32.qf16=vsub(Vu32.hf,Vv32.qf16)
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vsub_VhfVqf16(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VS Execution Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vsub_VhfVqf16(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VS
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Vqf16_vsub_VhfVqf16(Vu, Vv) \
|
||||
@@ -5386,8 +5407,9 @@
|
||||
#if __HVX_ARCH__ >= 81
|
||||
/* ==========================================================================
|
||||
Assembly Syntax: Vd32.qf32=vsub(Vu32.sf,Vv32.qf32)
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vsub_VsfVqf32(HVX_Vector Vu,
|
||||
HVX_Vector Vv) Instruction Type: CVI_VS Execution Slots: SLOT0123
|
||||
C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vsub_VsfVqf32(HVX_Vector Vu, HVX_Vector Vv)
|
||||
Instruction Type: CVI_VS
|
||||
Execution Slots: SLOT0123
|
||||
========================================================================== */
|
||||
|
||||
#define Q6_Vqf32_vsub_VsfVqf32(Vu, Vv) \
|
||||
|
||||
Vendored
+4
-2
@@ -58,6 +58,10 @@
|
||||
|
||||
#include <avx512bitalgintrin.h>
|
||||
|
||||
#include <avx512bmmintrin.h>
|
||||
|
||||
#include <avx512bmmvlintrin.h>
|
||||
|
||||
#include <avx512cdintrin.h>
|
||||
|
||||
#include <avx512vpopcntdqintrin.h>
|
||||
@@ -479,8 +483,6 @@ _storebe_i64(void * __P, long long __D) {
|
||||
|
||||
#include <amxavx512intrin.h>
|
||||
|
||||
#include <amxtf32intrin.h>
|
||||
|
||||
#include <avx512vp2intersectintrin.h>
|
||||
|
||||
#include <avx512vlvp2intersectintrin.h>
|
||||
|
||||
Vendored
+20
-10
@@ -44,8 +44,10 @@
|
||||
|
||||
#if __x86_64__
|
||||
#define __LPTRINT_TYPE__ __int64
|
||||
#define __IPTRINT_TYPE__ __int64
|
||||
#else
|
||||
#define __LPTRINT_TYPE__ long
|
||||
#define __IPTRINT_TYPE__ int
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
@@ -95,12 +97,12 @@ void __outdword(unsigned short, unsigned long);
|
||||
void __outdwordstring(unsigned short, unsigned long *, unsigned long);
|
||||
void __outword(unsigned short, unsigned short);
|
||||
void __outwordstring(unsigned short, unsigned short *, unsigned long);
|
||||
unsigned long __readcr0(void);
|
||||
unsigned long __readcr2(void);
|
||||
unsigned __LPTRINT_TYPE__ __readcr0(void);
|
||||
unsigned __LPTRINT_TYPE__ __readcr2(void);
|
||||
unsigned __LPTRINT_TYPE__ __readcr3(void);
|
||||
unsigned __LPTRINT_TYPE__ __readcr4(void);
|
||||
unsigned __int64 __readcr8(void);
|
||||
unsigned int __readdr(unsigned int);
|
||||
unsigned __LPTRINT_TYPE__ __readcr8(void);
|
||||
unsigned __IPTRINT_TYPE__ __readdr(unsigned int);
|
||||
#ifdef __i386__
|
||||
unsigned char __readfsbyte(unsigned long);
|
||||
unsigned short __readfsword(unsigned long);
|
||||
@@ -126,11 +128,12 @@ unsigned __int64 __ull_rshift(unsigned __int64, int);
|
||||
void __vmx_off(void);
|
||||
void __vmx_vmptrst(unsigned __int64 *);
|
||||
void __wbinvd(void);
|
||||
void __writecr0(unsigned int);
|
||||
void __writecr3(unsigned __INTPTR_TYPE__);
|
||||
void __writecr4(unsigned __INTPTR_TYPE__);
|
||||
void __writecr8(unsigned __int64);
|
||||
void __writedr(unsigned int, unsigned int);
|
||||
void __writecr0(unsigned __IPTRINT_TYPE__);
|
||||
void __writecr2(unsigned __IPTRINT_TYPE__);
|
||||
void __writecr3(unsigned __IPTRINT_TYPE__);
|
||||
void __writecr4(unsigned __IPTRINT_TYPE__);
|
||||
void __writecr8(unsigned __IPTRINT_TYPE__);
|
||||
void __writedr(unsigned int, unsigned __IPTRINT_TYPE__);
|
||||
void __writefsbyte(unsigned long, unsigned char);
|
||||
void __writefsdword(unsigned long, unsigned long);
|
||||
void __writefsqword(unsigned long, unsigned __int64);
|
||||
@@ -374,6 +377,9 @@ static __inline__ void __DEFAULT_FN_ATTRS __nop(void) {
|
||||
\*----------------------------------------------------------------------------*/
|
||||
#if defined(__aarch64__) || defined(__arm64ec__)
|
||||
unsigned __int64 __getReg(int);
|
||||
double __getRegFp(int _Reg);
|
||||
void __setReg(int, unsigned __int64);
|
||||
void __setRegFp(int, double);
|
||||
unsigned char _interlockedbittestandreset_acq(long volatile *, long);
|
||||
unsigned char _interlockedbittestandreset_nf(long volatile *, long);
|
||||
unsigned char _interlockedbittestandreset_rel(long volatile *, long);
|
||||
@@ -438,10 +444,13 @@ unsigned int _CountLeadingSigns(long);
|
||||
unsigned int _CountLeadingSigns64(__int64);
|
||||
unsigned int _CountOneBits(unsigned long);
|
||||
unsigned int _CountOneBits64(unsigned __int64);
|
||||
unsigned int _CountTrailingZeros(unsigned long);
|
||||
unsigned int _CountTrailingZeros64(unsigned __int64);
|
||||
|
||||
unsigned int __hlt(unsigned int, ...);
|
||||
|
||||
void __cdecl __prefetch(const void *);
|
||||
void __cdecl __prefetch2(const void *, unsigned char);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -474,7 +483,7 @@ static __inline__ unsigned __LPTRINT_TYPE__ __DEFAULT_FN_ATTRS __readcr3(void) {
|
||||
}
|
||||
|
||||
static __inline__ void __DEFAULT_FN_ATTRS
|
||||
__writecr3(unsigned __INTPTR_TYPE__ __cr3_val) {
|
||||
__writecr3(unsigned __IPTRINT_TYPE__ __cr3_val) {
|
||||
__asm__ ("mov {%0, %%cr3|cr3, %0}" : : "r"(__cr3_val) : "memory");
|
||||
}
|
||||
#endif
|
||||
@@ -484,6 +493,7 @@ __writecr3(unsigned __INTPTR_TYPE__ __cr3_val) {
|
||||
#endif
|
||||
|
||||
#undef __LPTRINT_TYPE__
|
||||
#undef __IPTRINT_TYPE__
|
||||
|
||||
#undef __DEFAULT_FN_ATTRS
|
||||
|
||||
|
||||
Vendored
+2
@@ -120,10 +120,12 @@ extern __inline int
|
||||
|
||||
#define __ibar(/*ui15*/ _1) __builtin_loongarch_ibar((_1))
|
||||
|
||||
#ifdef __loongarch_hard_float
|
||||
#define __movfcsr2gr(/*ui5*/ _1) __builtin_loongarch_movfcsr2gr((_1));
|
||||
|
||||
#define __movgr2fcsr(/*ui5*/ _1, _2) \
|
||||
__builtin_loongarch_movgr2fcsr((_1), (unsigned int)_2);
|
||||
#endif
|
||||
|
||||
#define __syscall(/*ui15*/ _1) __builtin_loongarch_syscall((_1))
|
||||
|
||||
|
||||
+3
-2
@@ -9,13 +9,14 @@
|
||||
#ifndef __CLANG_LLVM_LIBC_WRAPPERS_CTYPE_H__
|
||||
#define __CLANG_LLVM_LIBC_WRAPPERS_CTYPE_H__
|
||||
|
||||
#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__)
|
||||
#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__) && \
|
||||
!defined(__SPIRV__)
|
||||
#error "This file is for GPU offloading compilation only"
|
||||
#endif
|
||||
|
||||
#include_next <ctype.h>
|
||||
|
||||
#if defined(__HIP__) || defined(__CUDA__)
|
||||
#if defined(__HIP__) || defined(__CUDA__) || defined(__SPIRV__)
|
||||
#define __LIBC_ATTRS __attribute__((device))
|
||||
#else
|
||||
#define __LIBC_ATTRS
|
||||
|
||||
+3
-2
@@ -9,13 +9,14 @@
|
||||
#ifndef __CLANG_LLVM_LIBC_WRAPPERS_STRING_H__
|
||||
#define __CLANG_LLVM_LIBC_WRAPPERS_STRING_H__
|
||||
|
||||
#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__)
|
||||
#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__) && \
|
||||
!defined(__SPIRV__)
|
||||
#error "This file is for GPU offloading compilation only"
|
||||
#endif
|
||||
|
||||
#include_next <string.h>
|
||||
|
||||
#if defined(__HIP__) || defined(__CUDA__)
|
||||
#if defined(__HIP__) || defined(__CUDA__) || defined(__SPIRV__)
|
||||
#define __LIBC_ATTRS __attribute__((device))
|
||||
#else
|
||||
#define __LIBC_ATTRS
|
||||
|
||||
Vendored
+16
-1
@@ -21,10 +21,19 @@ module _Builtin_intrinsics [system] [extern_c] {
|
||||
export *
|
||||
}
|
||||
|
||||
explicit module bf16 {
|
||||
header "arm_bf16.h"
|
||||
export *
|
||||
}
|
||||
|
||||
explicit module fp16 {
|
||||
header "arm_fp16.h"
|
||||
export *
|
||||
}
|
||||
|
||||
explicit module neon {
|
||||
requires neon
|
||||
header "arm_neon.h"
|
||||
header "arm_fp16.h"
|
||||
export *
|
||||
}
|
||||
|
||||
@@ -41,6 +50,12 @@ module _Builtin_intrinsics [system] [extern_c] {
|
||||
|
||||
header "arm64intr.h"
|
||||
export *
|
||||
|
||||
explicit module neon {
|
||||
requires neon
|
||||
header "arm64_neon.h"
|
||||
export *
|
||||
}
|
||||
}
|
||||
|
||||
explicit module intel {
|
||||
|
||||
@@ -55,6 +55,20 @@ extern "C" {
|
||||
#pragma omp end declare variant
|
||||
#endif
|
||||
|
||||
#ifdef __SPIRV__
|
||||
#pragma omp begin declare variant match( \
|
||||
device = {arch(spirv64)}, implementation = {extension(match_any)})
|
||||
|
||||
#define __OPENMP_SPIRV__
|
||||
|
||||
/// Include declarations for libdevice functions.
|
||||
#include <__clang_spirv_libdevice_declares.h>
|
||||
|
||||
#undef __OPENMP_SPIRV__
|
||||
|
||||
#pragma omp end declare variant
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
Vendored
+7
-1
@@ -29,6 +29,12 @@
|
||||
#undef __OPENMP_AMDGCN__
|
||||
#endif // __AMDGCN__
|
||||
|
||||
#ifdef __SPIRV__
|
||||
#define __OPENMP_SPIRV__
|
||||
#include <__clang_cuda_complex_builtins.h>
|
||||
#undef __OPENMP_SPIRV__
|
||||
#endif // __SPIRV__
|
||||
|
||||
#endif
|
||||
|
||||
// Grab the host header too.
|
||||
@@ -45,7 +51,7 @@
|
||||
#ifndef _LIBCPP_STD_VER
|
||||
|
||||
#pragma omp begin declare variant match( \
|
||||
device = {arch(amdgcn, nvptx, nvptx64)}, \
|
||||
device = {arch(amdgcn, nvptx, nvptx64, spirv64)}, \
|
||||
implementation = {extension(match_any, allow_templates)})
|
||||
|
||||
#include <complex_cmath.h>
|
||||
|
||||
+6
@@ -29,6 +29,12 @@
|
||||
#undef __OPENMP_AMDGCN__
|
||||
#endif
|
||||
|
||||
#ifdef __SPIRV__
|
||||
#define __OPENMP_SPIRV__
|
||||
#include <__clang_cuda_complex_builtins.h>
|
||||
#undef __OPENMP_SPIRV__
|
||||
#endif // __SPIRV__
|
||||
|
||||
#endif
|
||||
|
||||
// Grab the host header too.
|
||||
|
||||
Vendored
+10
@@ -58,4 +58,14 @@
|
||||
#pragma omp end declare variant
|
||||
#endif
|
||||
|
||||
#ifdef __SPIRV__
|
||||
#pragma omp begin declare variant match(device = {arch(spirv64)})
|
||||
|
||||
#define __OPENMP_SPIRV__
|
||||
#include <__clang_spirv_math.h>
|
||||
#undef __OPENMP_SPIRV__
|
||||
|
||||
#pragma omp end declare variant
|
||||
#endif
|
||||
|
||||
#endif
|
||||
Vendored
+90
-12
@@ -38,8 +38,7 @@ typedef enum {
|
||||
The extra data is always 0. */
|
||||
ptrauth_key_function_pointer = ptrauth_key_process_independent_code,
|
||||
|
||||
/* The key used to sign C++ v-table pointers.
|
||||
The extra data is always 0. */
|
||||
/* The key used to sign pointers to C++ v-tables. */
|
||||
ptrauth_key_cxx_vtable_pointer = ptrauth_key_process_independent_data,
|
||||
|
||||
/* The key used to sign metadata pointers to Objective-C method-lists. */
|
||||
@@ -178,6 +177,56 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
|
||||
__builtin_ptrauth_auth_and_resign(__value, __old_key, __old_data, __new_key, \
|
||||
__new_data)
|
||||
|
||||
/* Authenticate a pointer using a PC-based signature scheme and resign
|
||||
it using a different scheme.
|
||||
|
||||
If the result is subsequently authenticated using the new scheme, that
|
||||
authentication is guaranteed to fail if and only if the initial
|
||||
authentication failed.
|
||||
|
||||
The value must be an expression of pointer type.
|
||||
The key must be a constant expression of type ptrauth_key.
|
||||
The extra data must be an expression of pointer or integer type;
|
||||
if an integer, it will be coerced to ptrauth_extra_data_t.
|
||||
The oldpc must be an expression of pointer or integer type representing
|
||||
the PC value where the original signature was created.
|
||||
The result will have the same type as the original value.
|
||||
|
||||
This operation is guaranteed to not leave the intermediate value
|
||||
available for attack before it is re-signed.
|
||||
|
||||
Do not pass a null pointer to this function. A null pointer
|
||||
will not successfully authenticate. */
|
||||
#define ptrauth_auth_with_pc_and_resign(__value, __old_key, __old_data, \
|
||||
__old_pc, __new_key, __new_data) \
|
||||
__builtin_ptrauth_auth_with_pc_and_resign(__value, __old_key, __old_data, \
|
||||
__old_pc, __new_key, __new_data)
|
||||
|
||||
/* Authenticate a pointer using one scheme, load 32bit value at offset addend
|
||||
from the pointer, and add this value to the pointer, sign using specified
|
||||
scheme.
|
||||
|
||||
If the result is subsequently authenticated using the new scheme, that
|
||||
authentication is guaranteed to fail if and only if the initial
|
||||
authentication failed.
|
||||
|
||||
The value must be an expression of pointer type.
|
||||
The key must be a constant expression of type ptrauth_key.
|
||||
The extra data must be an expression of pointer or integer type;
|
||||
if an integer, it will be coerced to ptrauth_extra_data_t.
|
||||
The addend must be an immediate ptrdiff_t value.
|
||||
The result will have the same type as the original value.
|
||||
|
||||
This operation is guaranteed to not leave the intermediate value
|
||||
available for attack before it is re-signed.
|
||||
|
||||
Do not pass a null pointer to this function. A null pointer
|
||||
will not successfully authenticate. */
|
||||
#define ptrauth_auth_load_relative_and_sign(__value, __old_key, __old_data, \
|
||||
__new_key, __new_data, __offset) \
|
||||
__builtin_ptrauth_auth_load_relative_and_sign( \
|
||||
__value, __old_key, __old_data, __new_key, __new_data, __offset)
|
||||
|
||||
/* Authenticate a pointer using one scheme and resign it as a C
|
||||
function pointer.
|
||||
|
||||
@@ -324,30 +373,38 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
|
||||
__ptrauth(ptrauth_key_objc_class_ro_pointer, 1, \
|
||||
__ptrauth_objc_class_ro_discriminator)
|
||||
|
||||
#if __has_feature(ptrauth_init_fini_address_discrimination)
|
||||
#define __ptrauth_init_fini_pointer \
|
||||
__ptrauth(ptrauth_key_init_fini_pointer, 1, __ptrauth_init_fini_discriminator)
|
||||
#else
|
||||
#define __ptrauth_init_fini_pointer \
|
||||
__ptrauth(ptrauth_key_init_fini_pointer, 0, __ptrauth_init_fini_discriminator)
|
||||
#endif
|
||||
|
||||
#else
|
||||
|
||||
#define ptrauth_strip(__value, __key) \
|
||||
({ \
|
||||
__extension__({ \
|
||||
(void)__key; \
|
||||
__value; \
|
||||
})
|
||||
|
||||
#define ptrauth_blend_discriminator(__pointer, __integer) \
|
||||
({ \
|
||||
__extension__({ \
|
||||
(void)__pointer; \
|
||||
(void)__integer; \
|
||||
((ptrauth_extra_data_t)0); \
|
||||
})
|
||||
|
||||
#define ptrauth_sign_constant(__value, __key, __data) \
|
||||
({ \
|
||||
__extension__({ \
|
||||
(void)__key; \
|
||||
(void)__data; \
|
||||
__value; \
|
||||
})
|
||||
|
||||
#define ptrauth_sign_unauthenticated(__value, __key, __data) \
|
||||
({ \
|
||||
__extension__({ \
|
||||
(void)__key; \
|
||||
(void)__data; \
|
||||
__value; \
|
||||
@@ -355,7 +412,7 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
|
||||
|
||||
#define ptrauth_auth_and_resign(__value, __old_key, __old_data, __new_key, \
|
||||
__new_data) \
|
||||
({ \
|
||||
__extension__({ \
|
||||
(void)__old_key; \
|
||||
(void)__old_data; \
|
||||
(void)__new_key; \
|
||||
@@ -363,22 +420,44 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
|
||||
__value; \
|
||||
})
|
||||
|
||||
#define ptrauth_auth_with_pc_and_resign(__value, __old_key, __old_data, \
|
||||
__old_pc, __new_key, __new_data) \
|
||||
__extension__({ \
|
||||
(void)__old_key; \
|
||||
(void)__old_data; \
|
||||
(void)__old_pc; \
|
||||
(void)__new_key; \
|
||||
(void)__new_data; \
|
||||
__value; \
|
||||
})
|
||||
|
||||
#define ptrauth_auth_load_relative_and_sign(__value, __old_key, __old_data, \
|
||||
__new_key, __new_data, __offset) \
|
||||
__extension__({ \
|
||||
(void)__old_key; \
|
||||
(void)__old_data; \
|
||||
(void)__new_key; \
|
||||
(void)__new_data; \
|
||||
const char *__value_tmp = (const char *)(__value); \
|
||||
(void *)(__value_tmp + *(const int *)(__value_tmp + (__offset))); \
|
||||
})
|
||||
|
||||
#define ptrauth_auth_function(__value, __old_key, __old_data) \
|
||||
({ \
|
||||
__extension__({ \
|
||||
(void)__old_key; \
|
||||
(void)__old_data; \
|
||||
__value; \
|
||||
})
|
||||
|
||||
#define ptrauth_auth_data(__value, __old_key, __old_data) \
|
||||
({ \
|
||||
__extension__({ \
|
||||
(void)__old_key; \
|
||||
(void)__old_data; \
|
||||
__value; \
|
||||
})
|
||||
|
||||
#define ptrauth_string_discriminator(__string) \
|
||||
({ \
|
||||
__extension__({ \
|
||||
(void)__string; \
|
||||
((ptrauth_extra_data_t)0); \
|
||||
})
|
||||
@@ -388,13 +467,12 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
|
||||
((ptrauth_extra_data_t)0)
|
||||
|
||||
#define ptrauth_sign_generic_data(__value, __data) \
|
||||
({ \
|
||||
__extension__({ \
|
||||
(void)__value; \
|
||||
(void)__data; \
|
||||
((ptrauth_generic_signature_t)0); \
|
||||
})
|
||||
|
||||
|
||||
#define ptrauth_cxx_vtable_pointer(key, address_discrimination, \
|
||||
extra_discrimination...)
|
||||
|
||||
|
||||
Vendored
+7
@@ -16,6 +16,13 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define __riscv_intrinsic_b 1
|
||||
#define __riscv_intrinsic_zbb 1
|
||||
#define __riscv_intrinsic_zbc 1
|
||||
#define __riscv_intrinsic_zbkb 1
|
||||
#define __riscv_intrinsic_zbkc 1
|
||||
#define __riscv_intrinsic_zbkx 1
|
||||
|
||||
#if defined(__riscv_zbb)
|
||||
static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
|
||||
__riscv_orc_b_32(uint32_t __x) {
|
||||
|
||||
Vendored
+2
@@ -16,6 +16,8 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define __riscv_intrinsic_xcvalu 1
|
||||
|
||||
#if defined(__riscv_xcvalu)
|
||||
|
||||
#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
|
||||
|
||||
Vendored
+8
@@ -16,6 +16,14 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define __riscv_intrinsic_zkn 1
|
||||
#define __riscv_intrinsic_zknd 1
|
||||
#define __riscv_intrinsic_zkne 1
|
||||
#define __riscv_intrinsic_zknh 1
|
||||
#define __riscv_intrinsic_zks 1
|
||||
#define __riscv_intrinsic_zksed 1
|
||||
#define __riscv_intrinsic_zksh 1
|
||||
|
||||
#if defined(__riscv_zknd)
|
||||
#if __riscv_xlen == 32
|
||||
#define __riscv_aes32dsi(x, y, bs) __builtin_riscv_aes32dsi(x, y, bs)
|
||||
|
||||
Vendored
+2
@@ -14,6 +14,8 @@
|
||||
#error "This header is only meant to be used on riscv architecture"
|
||||
#endif
|
||||
|
||||
#define __riscv_intrinsic_xmipsexectl 1
|
||||
|
||||
#define __DEFAULT_FN_ATTRS \
|
||||
__attribute__((__always_inline__, __nodebug__, __target__("xmipsexectl")))
|
||||
|
||||
|
||||
Vendored
+3
@@ -16,6 +16,9 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define __riscv_intrinsic_xandesbfhcvt 1
|
||||
#define __riscv_intrinsic_xandesperf 1
|
||||
|
||||
#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
|
||||
|
||||
#if defined(__riscv_xandesperf)
|
||||
|
||||
Vendored
+2
@@ -10,6 +10,8 @@
|
||||
#ifndef __RISCV_NTLH_H
|
||||
#define __RISCV_NTLH_H
|
||||
|
||||
#define __riscv_intrinsic_zihintntl 1
|
||||
|
||||
#ifndef __riscv_zihintntl
|
||||
#error "NTLH intrinsics require the NTLH extension."
|
||||
#endif
|
||||
|
||||
Vendored
+644
@@ -0,0 +1,644 @@
|
||||
/*===---- riscv_packed_simd.h - RISC-V P intrinsics ------------------------===
|
||||
*
|
||||
* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
* See https://llvm.org/LICENSE.txt for license information.
|
||||
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
*
|
||||
*===-----------------------------------------------------------------------===
|
||||
*/
|
||||
|
||||
#ifndef __RISCV_PACKED_SIMD_H
|
||||
#define __RISCV_PACKED_SIMD_H
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#if defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* Packed SIMD Types */
|
||||
|
||||
typedef int8_t int8x4_t __attribute__((__vector_size__(4)));
|
||||
typedef uint8_t uint8x4_t __attribute__((__vector_size__(4)));
|
||||
typedef int16_t int16x2_t __attribute__((__vector_size__(4)));
|
||||
typedef uint16_t uint16x2_t __attribute__((__vector_size__(4)));
|
||||
|
||||
typedef int8_t int8x8_t __attribute__((__vector_size__(8)));
|
||||
typedef uint8_t uint8x8_t __attribute__((__vector_size__(8)));
|
||||
typedef int16_t int16x4_t __attribute__((__vector_size__(8)));
|
||||
typedef uint16_t uint16x4_t __attribute__((__vector_size__(8)));
|
||||
typedef int32_t int32x2_t __attribute__((__vector_size__(8)));
|
||||
typedef uint32_t uint32x2_t __attribute__((__vector_size__(8)));
|
||||
|
||||
#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
|
||||
|
||||
#define __packed_splat2(ty, x) ((ty){(x), (x)})
|
||||
#define __packed_splat4(ty, x) ((ty){(x), (x), (x), (x)})
|
||||
#define __packed_splat8(ty, x) ((ty){(x), (x), (x), (x), (x), (x), (x), (x)})
|
||||
|
||||
#define __packed_splat(name, ty, scalar_ty, splat) \
|
||||
static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(scalar_ty __x) { \
|
||||
return splat(ty, __x); \
|
||||
}
|
||||
|
||||
#define __packed_shift(name, ty, op, mask) \
|
||||
static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \
|
||||
unsigned __rs2) { \
|
||||
return __rs1 op(__rs2 & (mask)); \
|
||||
}
|
||||
#define __packed_shift8(name, ty, op) __packed_shift(name, ty, op, 0x7)
|
||||
#define __packed_shift16(name, ty, op) __packed_shift(name, ty, op, 0xf)
|
||||
#define __packed_shift32(name, ty, op) __packed_shift(name, ty, op, 0x1f)
|
||||
|
||||
#define __packed_scalar_binary_op(name, ty, scalar_ty, op, splat) \
|
||||
static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \
|
||||
scalar_ty __rs2) { \
|
||||
return __rs1 op splat(ty, __rs2); \
|
||||
}
|
||||
|
||||
#define __packed_binary_op(name, ty, op) \
|
||||
static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, ty __rs2) { \
|
||||
return __rs1 op __rs2; \
|
||||
}
|
||||
|
||||
#define __packed_unary_op(name, ty, op) \
|
||||
static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \
|
||||
return op __rs1; \
|
||||
}
|
||||
|
||||
#define __packed_binary_builtin(name, ty, builtin) \
|
||||
static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, ty __rs2) { \
|
||||
return builtin(__rs1, __rs2); \
|
||||
}
|
||||
|
||||
#define __packed_sh1add(name, ty) \
|
||||
static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, ty __rs2) { \
|
||||
return (__rs1 << 1) + __rs2; \
|
||||
}
|
||||
|
||||
/* TODO: switch to sadd_sat(__builtin_elementwise_shl_sat(a, 1), b) once a
|
||||
* generic elementwise shl_sat builtin exists. sadd_sat(a, a) is equivalent
|
||||
* for signed types and the backend's saturating_shl1 PatFrags matches both
|
||||
* shapes. */
|
||||
#define __packed_sh1sadd(name, ty) \
|
||||
static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, ty __rs2) { \
|
||||
return __builtin_elementwise_add_sat( \
|
||||
__builtin_elementwise_add_sat(__rs1, __rs1), __rs2); \
|
||||
}
|
||||
|
||||
#define __packed_cmp(name, ty, rty, op) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \
|
||||
ty __rs2) { \
|
||||
return (rty)(__rs1 op __rs2); \
|
||||
}
|
||||
|
||||
#define __packed_pabs(name, ty, rty) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \
|
||||
return (rty)__builtin_elementwise_abs(__rs1); \
|
||||
}
|
||||
|
||||
#define __packed_binary_builtin_cast(name, ty, rty, builtin) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \
|
||||
ty __rs2) { \
|
||||
return (rty)builtin(__rs1, __rs2); \
|
||||
}
|
||||
|
||||
#define __packed_reduction(name, rty, ty, builtin) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \
|
||||
rty __rs2) { \
|
||||
return builtin(__rs1, __rs2); \
|
||||
}
|
||||
|
||||
#define __packed_merge_builtin(name, ty, mask_ty, builtin) \
|
||||
static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, ty __rs2, \
|
||||
mask_ty __rd) { \
|
||||
return (ty)builtin(__rs1, __rs2, __rd); \
|
||||
}
|
||||
|
||||
#define __packed_psabs(name, ty, builtin) \
|
||||
static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \
|
||||
return builtin(__rs1); \
|
||||
}
|
||||
|
||||
#define __packed_widen_convert(name, rty, ty) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \
|
||||
return __builtin_convertvector(__rs1, rty); \
|
||||
}
|
||||
#define __packed_widen_high2(name, rty, ty) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \
|
||||
return (rty)__builtin_shufflevector((ty){0}, __rs1, 0, 2, 1, 3); \
|
||||
}
|
||||
#define __packed_widen_high4(name, rty, ty) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \
|
||||
return (rty)__builtin_shufflevector((ty){0}, __rs1, 0, 4, 1, 5, 2, 6, 3, \
|
||||
7); \
|
||||
}
|
||||
|
||||
/* Packed Reverse: reverse the order of the elements. Lowered to a single
|
||||
* rev8/rev16/ppairoe.* by the backend's packed reverse-shuffle handling. */
|
||||
#define __packed_reverse2(name, ty) \
|
||||
static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \
|
||||
return __builtin_shufflevector(__rs1, __rs1, 1, 0); \
|
||||
}
|
||||
#define __packed_reverse4(name, ty) \
|
||||
static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \
|
||||
return __builtin_shufflevector(__rs1, __rs1, 3, 2, 1, 0); \
|
||||
}
|
||||
#define __packed_reverse8(name, ty) \
|
||||
static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \
|
||||
return __builtin_shufflevector(__rs1, __rs1, 7, 6, 5, 4, 3, 2, 1, 0); \
|
||||
}
|
||||
|
||||
#define __packed_zip2(name, rty, ty) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \
|
||||
ty __rs2) { \
|
||||
return __builtin_shufflevector(__rs1, __rs2, 0, 2, 1, 3); \
|
||||
}
|
||||
#define __packed_zip4(name, rty, ty) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \
|
||||
ty __rs2) { \
|
||||
return __builtin_shufflevector(__rs1, __rs2, 0, 4, 1, 5, 2, 6, 3, 7); \
|
||||
}
|
||||
#define __packed_unzipe2(name, rty, ty) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \
|
||||
return __builtin_shufflevector(__rs1, __rs1, 0, 2); \
|
||||
}
|
||||
#define __packed_unzipe4(name, rty, ty) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \
|
||||
return __builtin_shufflevector(__rs1, __rs1, 0, 2, 4, 6); \
|
||||
}
|
||||
#define __packed_unzipo2(name, rty, ty) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \
|
||||
return __builtin_shufflevector(__rs1, __rs1, 1, 3); \
|
||||
}
|
||||
#define __packed_unzipo4(name, rty, ty) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \
|
||||
return __builtin_shufflevector(__rs1, __rs1, 1, 3, 5, 7); \
|
||||
}
|
||||
|
||||
#define __packed_abdsum(name, rty, ty, builtin) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \
|
||||
ty __rs2) { \
|
||||
return builtin(__rs1, __rs2); \
|
||||
}
|
||||
|
||||
#define __packed_abdsum_acc(name, rty, ty, builtin) \
|
||||
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(rty __rd, ty __rs1, \
|
||||
ty __rs2) { \
|
||||
return builtin(__rd, __rs1, __rs2); \
|
||||
}
|
||||
|
||||
// clang-format off: macro call sites have no trailing semicolons, which
|
||||
// confuses clang-format into a deeply nested expression.
|
||||
|
||||
/* Packed Splat (32-bit) */
|
||||
__packed_splat(pmv_s_u8x4, uint8x4_t, uint8_t, __packed_splat4)
|
||||
__packed_splat(pmv_s_i8x4, int8x4_t, int8_t, __packed_splat4)
|
||||
__packed_splat(pmv_s_u16x2, uint16x2_t, uint16_t, __packed_splat2)
|
||||
__packed_splat(pmv_s_i16x2, int16x2_t, int16_t, __packed_splat2)
|
||||
|
||||
/* Packed Splat (64-bit) */
|
||||
__packed_splat(pmv_s_u8x8, uint8x8_t, uint8_t, __packed_splat8)
|
||||
__packed_splat(pmv_s_i8x8, int8x8_t, int8_t, __packed_splat8)
|
||||
__packed_splat(pmv_s_u16x4, uint16x4_t, uint16_t, __packed_splat4)
|
||||
__packed_splat(pmv_s_i16x4, int16x4_t, int16_t, __packed_splat4)
|
||||
__packed_splat(pmv_s_u32x2, uint32x2_t, uint32_t, __packed_splat2)
|
||||
__packed_splat(pmv_s_i32x2, int32x2_t, int32_t, __packed_splat2)
|
||||
|
||||
/* Packed Addition and Subtraction (32-bit) */
|
||||
__packed_binary_op(padd_i8x4, int8x4_t, +)
|
||||
__packed_binary_op(padd_u8x4, uint8x4_t, +)
|
||||
__packed_binary_op(padd_i16x2, int16x2_t, +)
|
||||
__packed_binary_op(padd_u16x2, uint16x2_t, +)
|
||||
__packed_binary_op(psub_i8x4, int8x4_t, -)
|
||||
__packed_binary_op(psub_u8x4, uint8x4_t, -)
|
||||
__packed_binary_op(psub_i16x2, int16x2_t, -)
|
||||
__packed_binary_op(psub_u16x2, uint16x2_t, -)
|
||||
__packed_unary_op(pneg_i8x4, int8x4_t, -)
|
||||
__packed_unary_op(pneg_i16x2, int16x2_t, -)
|
||||
|
||||
/* Packed Addition and Subtraction (64-bit) */
|
||||
__packed_binary_op(padd_i8x8, int8x8_t, +)
|
||||
__packed_binary_op(padd_u8x8, uint8x8_t, +)
|
||||
__packed_binary_op(padd_i16x4, int16x4_t, +)
|
||||
__packed_binary_op(padd_u16x4, uint16x4_t, +)
|
||||
__packed_binary_op(padd_i32x2, int32x2_t, +)
|
||||
__packed_binary_op(padd_u32x2, uint32x2_t, +)
|
||||
__packed_binary_op(psub_i8x8, int8x8_t, -)
|
||||
__packed_binary_op(psub_u8x8, uint8x8_t, -)
|
||||
__packed_binary_op(psub_i16x4, int16x4_t, -)
|
||||
__packed_binary_op(psub_u16x4, uint16x4_t, -)
|
||||
__packed_binary_op(psub_i32x2, int32x2_t, -)
|
||||
__packed_binary_op(psub_u32x2, uint32x2_t, -)
|
||||
__packed_unary_op(pneg_i8x8, int8x8_t, -)
|
||||
__packed_unary_op(pneg_i16x4, int16x4_t, -)
|
||||
__packed_unary_op(pneg_i32x2, int32x2_t, -)
|
||||
|
||||
/* Packed Addition with Scalar (32-bit) */
|
||||
__packed_scalar_binary_op(padd_s_u8x4, uint8x4_t, uint8_t, +, __packed_splat4)
|
||||
__packed_scalar_binary_op(padd_s_i8x4, int8x4_t, int8_t, +, __packed_splat4)
|
||||
__packed_scalar_binary_op(padd_s_u16x2, uint16x2_t, uint16_t, +,
|
||||
__packed_splat2)
|
||||
__packed_scalar_binary_op(padd_s_i16x2, int16x2_t, int16_t, +,
|
||||
__packed_splat2)
|
||||
|
||||
/* Packed Addition with Scalar (64-bit) */
|
||||
__packed_scalar_binary_op(padd_s_u8x8, uint8x8_t, uint8_t, +, __packed_splat8)
|
||||
__packed_scalar_binary_op(padd_s_i8x8, int8x8_t, int8_t, +, __packed_splat8)
|
||||
__packed_scalar_binary_op(padd_s_u16x4, uint16x4_t, uint16_t, +,
|
||||
__packed_splat4)
|
||||
__packed_scalar_binary_op(padd_s_i16x4, int16x4_t, int16_t, +,
|
||||
__packed_splat4)
|
||||
__packed_scalar_binary_op(padd_s_u32x2, uint32x2_t, uint32_t, +,
|
||||
__packed_splat2)
|
||||
__packed_scalar_binary_op(padd_s_i32x2, int32x2_t, int32_t, +,
|
||||
__packed_splat2)
|
||||
|
||||
/* Packed Saturating Addition and Subtraction (32-bit) */
|
||||
__packed_binary_builtin(psadd_i8x4, int8x4_t, __builtin_elementwise_add_sat)
|
||||
__packed_binary_builtin(psadd_i16x2, int16x2_t, __builtin_elementwise_add_sat)
|
||||
__packed_binary_builtin(psaddu_u8x4, uint8x4_t, __builtin_elementwise_add_sat)
|
||||
__packed_binary_builtin(psaddu_u16x2, uint16x2_t, __builtin_elementwise_add_sat)
|
||||
__packed_binary_builtin(pssub_i8x4, int8x4_t, __builtin_elementwise_sub_sat)
|
||||
__packed_binary_builtin(pssub_i16x2, int16x2_t, __builtin_elementwise_sub_sat)
|
||||
__packed_binary_builtin(pssubu_u8x4, uint8x4_t, __builtin_elementwise_sub_sat)
|
||||
__packed_binary_builtin(pssubu_u16x2, uint16x2_t, __builtin_elementwise_sub_sat)
|
||||
|
||||
/* Packed Saturating Addition and Subtraction (64-bit) */
|
||||
__packed_binary_builtin(psadd_i8x8, int8x8_t, __builtin_elementwise_add_sat)
|
||||
__packed_binary_builtin(psadd_i16x4, int16x4_t, __builtin_elementwise_add_sat)
|
||||
__packed_binary_builtin(psadd_i32x2, int32x2_t, __builtin_elementwise_add_sat)
|
||||
__packed_binary_builtin(psaddu_u8x8, uint8x8_t, __builtin_elementwise_add_sat)
|
||||
__packed_binary_builtin(psaddu_u16x4, uint16x4_t, __builtin_elementwise_add_sat)
|
||||
__packed_binary_builtin(psaddu_u32x2, uint32x2_t, __builtin_elementwise_add_sat)
|
||||
__packed_binary_builtin(pssub_i8x8, int8x8_t, __builtin_elementwise_sub_sat)
|
||||
__packed_binary_builtin(pssub_i16x4, int16x4_t, __builtin_elementwise_sub_sat)
|
||||
__packed_binary_builtin(pssub_i32x2, int32x2_t, __builtin_elementwise_sub_sat)
|
||||
__packed_binary_builtin(pssubu_u8x8, uint8x8_t, __builtin_elementwise_sub_sat)
|
||||
__packed_binary_builtin(pssubu_u16x4, uint16x4_t, __builtin_elementwise_sub_sat)
|
||||
__packed_binary_builtin(pssubu_u32x2, uint32x2_t, __builtin_elementwise_sub_sat)
|
||||
|
||||
/* Packed Shift-Add (32-bit) */
|
||||
__packed_sh1add(psh1add_i16x2, int16x2_t)
|
||||
__packed_sh1add(psh1add_u16x2, uint16x2_t)
|
||||
__packed_sh1sadd(pssh1sadd_i16x2, int16x2_t)
|
||||
|
||||
/* Packed Shift-Add (64-bit) */
|
||||
__packed_sh1add(psh1add_i16x4, int16x4_t)
|
||||
__packed_sh1add(psh1add_u16x4, uint16x4_t)
|
||||
__packed_sh1add(psh1add_i32x2, int32x2_t)
|
||||
__packed_sh1add(psh1add_u32x2, uint32x2_t)
|
||||
__packed_sh1sadd(pssh1sadd_i16x4, int16x4_t)
|
||||
__packed_sh1sadd(pssh1sadd_i32x2, int32x2_t)
|
||||
|
||||
/* Packed Exchanged Addition and Subtraction (32-bit) */
|
||||
__packed_binary_builtin(pas_x_i16x2, int16x2_t, __builtin_riscv_pas_x_i16x2)
|
||||
__packed_binary_builtin(psa_x_i16x2, int16x2_t, __builtin_riscv_psa_x_i16x2)
|
||||
__packed_binary_builtin(psas_x_i16x2, int16x2_t, __builtin_riscv_psas_x_i16x2)
|
||||
__packed_binary_builtin(pssa_x_i16x2, int16x2_t, __builtin_riscv_pssa_x_i16x2)
|
||||
__packed_binary_builtin(paas_x_i16x2, int16x2_t, __builtin_riscv_paas_x_i16x2)
|
||||
__packed_binary_builtin(pasa_x_i16x2, int16x2_t, __builtin_riscv_pasa_x_i16x2)
|
||||
|
||||
/* Packed Exchanged Addition and Subtraction (64-bit) */
|
||||
__packed_binary_builtin(pas_x_i16x4, int16x4_t, __builtin_riscv_pas_x_i16x4)
|
||||
__packed_binary_builtin(psa_x_i16x4, int16x4_t, __builtin_riscv_psa_x_i16x4)
|
||||
__packed_binary_builtin(psas_x_i16x4, int16x4_t, __builtin_riscv_psas_x_i16x4)
|
||||
__packed_binary_builtin(pssa_x_i16x4, int16x4_t, __builtin_riscv_pssa_x_i16x4)
|
||||
__packed_binary_builtin(paas_x_i16x4, int16x4_t, __builtin_riscv_paas_x_i16x4)
|
||||
__packed_binary_builtin(pasa_x_i16x4, int16x4_t, __builtin_riscv_pasa_x_i16x4)
|
||||
__packed_binary_builtin(pas_x_i32x2, int32x2_t, __builtin_riscv_pas_x_i32x2)
|
||||
__packed_binary_builtin(psa_x_i32x2, int32x2_t, __builtin_riscv_psa_x_i32x2)
|
||||
__packed_binary_builtin(psas_x_i32x2, int32x2_t, __builtin_riscv_psas_x_i32x2)
|
||||
__packed_binary_builtin(pssa_x_i32x2, int32x2_t, __builtin_riscv_pssa_x_i32x2)
|
||||
__packed_binary_builtin(paas_x_i32x2, int32x2_t, __builtin_riscv_paas_x_i32x2)
|
||||
__packed_binary_builtin(pasa_x_i32x2, int32x2_t, __builtin_riscv_pasa_x_i32x2)
|
||||
|
||||
/* Packed Minimum and Maximum (32-bit) */
|
||||
__packed_binary_builtin(pmin_i8x4, int8x4_t, __builtin_elementwise_min)
|
||||
__packed_binary_builtin(pmin_i16x2, int16x2_t, __builtin_elementwise_min)
|
||||
__packed_binary_builtin(pminu_u8x4, uint8x4_t, __builtin_elementwise_min)
|
||||
__packed_binary_builtin(pminu_u16x2, uint16x2_t, __builtin_elementwise_min)
|
||||
__packed_binary_builtin(pmax_i8x4, int8x4_t, __builtin_elementwise_max)
|
||||
__packed_binary_builtin(pmax_i16x2, int16x2_t, __builtin_elementwise_max)
|
||||
__packed_binary_builtin(pmaxu_u8x4, uint8x4_t, __builtin_elementwise_max)
|
||||
__packed_binary_builtin(pmaxu_u16x2, uint16x2_t, __builtin_elementwise_max)
|
||||
|
||||
/* Packed Minimum and Maximum (64-bit) */
|
||||
__packed_binary_builtin(pmin_i8x8, int8x8_t, __builtin_elementwise_min)
|
||||
__packed_binary_builtin(pmin_i16x4, int16x4_t, __builtin_elementwise_min)
|
||||
__packed_binary_builtin(pmin_i32x2, int32x2_t, __builtin_elementwise_min)
|
||||
__packed_binary_builtin(pminu_u8x8, uint8x8_t, __builtin_elementwise_min)
|
||||
__packed_binary_builtin(pminu_u16x4, uint16x4_t, __builtin_elementwise_min)
|
||||
__packed_binary_builtin(pminu_u32x2, uint32x2_t, __builtin_elementwise_min)
|
||||
__packed_binary_builtin(pmax_i8x8, int8x8_t, __builtin_elementwise_max)
|
||||
__packed_binary_builtin(pmax_i16x4, int16x4_t, __builtin_elementwise_max)
|
||||
__packed_binary_builtin(pmax_i32x2, int32x2_t, __builtin_elementwise_max)
|
||||
__packed_binary_builtin(pmaxu_u8x8, uint8x8_t, __builtin_elementwise_max)
|
||||
__packed_binary_builtin(pmaxu_u16x4, uint16x4_t, __builtin_elementwise_max)
|
||||
__packed_binary_builtin(pmaxu_u32x2, uint32x2_t, __builtin_elementwise_max)
|
||||
|
||||
/* Packed Comparison (32-bit) */
|
||||
__packed_cmp(pmseq_i8x4_u8x4, int8x4_t, uint8x4_t, ==)
|
||||
__packed_cmp(pmseq_u8x4_u8x4, uint8x4_t, uint8x4_t, ==)
|
||||
__packed_cmp(pmsne_i8x4_u8x4, int8x4_t, uint8x4_t, !=)
|
||||
__packed_cmp(pmsne_u8x4_u8x4, uint8x4_t, uint8x4_t, !=)
|
||||
__packed_cmp(pmslt_u8x4, int8x4_t, uint8x4_t, <)
|
||||
__packed_cmp(pmsltu_u8x4, uint8x4_t, uint8x4_t, <)
|
||||
__packed_cmp(pmsgt_u8x4, int8x4_t, uint8x4_t, >)
|
||||
__packed_cmp(pmsgtu_u8x4, uint8x4_t, uint8x4_t, >)
|
||||
__packed_cmp(pmsge_u8x4, int8x4_t, uint8x4_t, >=)
|
||||
__packed_cmp(pmsgeu_u8x4, uint8x4_t, uint8x4_t, >=)
|
||||
__packed_cmp(pmsle_u8x4, int8x4_t, uint8x4_t, <=)
|
||||
__packed_cmp(pmsleu_u8x4, uint8x4_t, uint8x4_t, <=)
|
||||
__packed_cmp(pmseq_i16x2_u16x2, int16x2_t, uint16x2_t, ==)
|
||||
__packed_cmp(pmseq_u16x2_u16x2, uint16x2_t, uint16x2_t, ==)
|
||||
__packed_cmp(pmsne_i16x2_u16x2, int16x2_t, uint16x2_t, !=)
|
||||
__packed_cmp(pmsne_u16x2_u16x2, uint16x2_t, uint16x2_t, !=)
|
||||
__packed_cmp(pmslt_u16x2, int16x2_t, uint16x2_t, <)
|
||||
__packed_cmp(pmsltu_u16x2, uint16x2_t, uint16x2_t, <)
|
||||
__packed_cmp(pmsgt_u16x2, int16x2_t, uint16x2_t, >)
|
||||
__packed_cmp(pmsgtu_u16x2, uint16x2_t, uint16x2_t, >)
|
||||
__packed_cmp(pmsge_u16x2, int16x2_t, uint16x2_t, >=)
|
||||
__packed_cmp(pmsgeu_u16x2, uint16x2_t, uint16x2_t, >=)
|
||||
__packed_cmp(pmsle_u16x2, int16x2_t, uint16x2_t, <=)
|
||||
__packed_cmp(pmsleu_u16x2, uint16x2_t, uint16x2_t, <=)
|
||||
|
||||
/* Packed Comparison (64-bit) */
|
||||
__packed_cmp(pmseq_i8x8_u8x8, int8x8_t, uint8x8_t, ==)
|
||||
__packed_cmp(pmseq_u8x8_u8x8, uint8x8_t, uint8x8_t, ==)
|
||||
__packed_cmp(pmsne_i8x8_u8x8, int8x8_t, uint8x8_t, !=)
|
||||
__packed_cmp(pmsne_u8x8_u8x8, uint8x8_t, uint8x8_t, !=)
|
||||
__packed_cmp(pmslt_u8x8, int8x8_t, uint8x8_t, <)
|
||||
__packed_cmp(pmsltu_u8x8, uint8x8_t, uint8x8_t, <)
|
||||
__packed_cmp(pmsgt_u8x8, int8x8_t, uint8x8_t, >)
|
||||
__packed_cmp(pmsgtu_u8x8, uint8x8_t, uint8x8_t, >)
|
||||
__packed_cmp(pmsge_u8x8, int8x8_t, uint8x8_t, >=)
|
||||
__packed_cmp(pmsgeu_u8x8, uint8x8_t, uint8x8_t, >=)
|
||||
__packed_cmp(pmsle_u8x8, int8x8_t, uint8x8_t, <=)
|
||||
__packed_cmp(pmsleu_u8x8, uint8x8_t, uint8x8_t, <=)
|
||||
__packed_cmp(pmseq_i16x4_u16x4, int16x4_t, uint16x4_t, ==)
|
||||
__packed_cmp(pmseq_u16x4_u16x4, uint16x4_t, uint16x4_t, ==)
|
||||
__packed_cmp(pmsne_i16x4_u16x4, int16x4_t, uint16x4_t, !=)
|
||||
__packed_cmp(pmsne_u16x4_u16x4, uint16x4_t, uint16x4_t, !=)
|
||||
__packed_cmp(pmslt_u16x4, int16x4_t, uint16x4_t, <)
|
||||
__packed_cmp(pmsltu_u16x4, uint16x4_t, uint16x4_t, <)
|
||||
__packed_cmp(pmsgt_u16x4, int16x4_t, uint16x4_t, >)
|
||||
__packed_cmp(pmsgtu_u16x4, uint16x4_t, uint16x4_t, >)
|
||||
__packed_cmp(pmsge_u16x4, int16x4_t, uint16x4_t, >=)
|
||||
__packed_cmp(pmsgeu_u16x4, uint16x4_t, uint16x4_t, >=)
|
||||
__packed_cmp(pmsle_u16x4, int16x4_t, uint16x4_t, <=)
|
||||
__packed_cmp(pmsleu_u16x4, uint16x4_t, uint16x4_t, <=)
|
||||
__packed_cmp(pmseq_i32x2_u32x2, int32x2_t, uint32x2_t, ==)
|
||||
__packed_cmp(pmseq_u32x2_u32x2, uint32x2_t, uint32x2_t, ==)
|
||||
__packed_cmp(pmsne_i32x2_u32x2, int32x2_t, uint32x2_t, !=)
|
||||
__packed_cmp(pmsne_u32x2_u32x2, uint32x2_t, uint32x2_t, !=)
|
||||
__packed_cmp(pmslt_u32x2, int32x2_t, uint32x2_t, <)
|
||||
__packed_cmp(pmsltu_u32x2, uint32x2_t, uint32x2_t, <)
|
||||
__packed_cmp(pmsgt_u32x2, int32x2_t, uint32x2_t, >)
|
||||
__packed_cmp(pmsgtu_u32x2, uint32x2_t, uint32x2_t, >)
|
||||
__packed_cmp(pmsge_u32x2, int32x2_t, uint32x2_t, >=)
|
||||
__packed_cmp(pmsgeu_u32x2, uint32x2_t, uint32x2_t, >=)
|
||||
__packed_cmp(pmsle_u32x2, int32x2_t, uint32x2_t, <=)
|
||||
__packed_cmp(pmsleu_u32x2, uint32x2_t, uint32x2_t, <=)
|
||||
|
||||
/* Packed Shifts (32-bit) */
|
||||
__packed_shift8(psll_s_u8x4, uint8x4_t, <<)
|
||||
__packed_shift8(psll_s_i8x4, int8x4_t, <<)
|
||||
__packed_shift16(psll_s_u16x2, uint16x2_t, <<)
|
||||
__packed_shift16(psll_s_i16x2, int16x2_t, <<)
|
||||
__packed_shift8(psrl_s_u8x4, uint8x4_t, >>)
|
||||
__packed_shift16(psrl_s_u16x2, uint16x2_t, >>)
|
||||
__packed_shift8(psra_s_i8x4, int8x4_t, >>)
|
||||
__packed_shift16(psra_s_i16x2, int16x2_t, >>)
|
||||
|
||||
/* Packed Shifts (64-bit) */
|
||||
__packed_shift8(psll_s_u8x8, uint8x8_t, <<)
|
||||
__packed_shift8(psll_s_i8x8, int8x8_t, <<)
|
||||
__packed_shift16(psll_s_u16x4, uint16x4_t, <<)
|
||||
__packed_shift16(psll_s_i16x4, int16x4_t, <<)
|
||||
__packed_shift32(psll_s_u32x2, uint32x2_t, <<)
|
||||
__packed_shift32(psll_s_i32x2, int32x2_t, <<)
|
||||
__packed_shift8(psrl_s_u8x8, uint8x8_t, >>)
|
||||
__packed_shift16(psrl_s_u16x4, uint16x4_t, >>)
|
||||
__packed_shift32(psrl_s_u32x2, uint32x2_t, >>)
|
||||
__packed_shift8(psra_s_i8x8, int8x8_t, >>)
|
||||
__packed_shift16(psra_s_i16x4, int16x4_t, >>)
|
||||
__packed_shift32(psra_s_i32x2, int32x2_t, >>)
|
||||
|
||||
/* Packed Logical Operations (32-bit) */
|
||||
__packed_binary_op(pand_i8x4, int8x4_t, &)
|
||||
__packed_binary_op(pand_u8x4, uint8x4_t, &)
|
||||
__packed_binary_op(pand_i16x2, int16x2_t, &)
|
||||
__packed_binary_op(pand_u16x2, uint16x2_t, &)
|
||||
__packed_binary_op(por_i8x4, int8x4_t, |)
|
||||
__packed_binary_op(por_u8x4, uint8x4_t, |)
|
||||
__packed_binary_op(por_i16x2, int16x2_t, |)
|
||||
__packed_binary_op(por_u16x2, uint16x2_t, |)
|
||||
__packed_binary_op(pxor_i8x4, int8x4_t, ^)
|
||||
__packed_binary_op(pxor_u8x4, uint8x4_t, ^)
|
||||
__packed_binary_op(pxor_i16x2, int16x2_t, ^)
|
||||
__packed_binary_op(pxor_u16x2, uint16x2_t, ^)
|
||||
__packed_unary_op(pnot_i8x4, int8x4_t, ~)
|
||||
__packed_unary_op(pnot_u8x4, uint8x4_t, ~)
|
||||
__packed_unary_op(pnot_i16x2, int16x2_t, ~)
|
||||
__packed_unary_op(pnot_u16x2, uint16x2_t, ~)
|
||||
|
||||
/* Packed Logical Operations (64-bit) */
|
||||
__packed_binary_op(pand_i8x8, int8x8_t, &)
|
||||
__packed_binary_op(pand_u8x8, uint8x8_t, &)
|
||||
__packed_binary_op(pand_i16x4, int16x4_t, &)
|
||||
__packed_binary_op(pand_u16x4, uint16x4_t, &)
|
||||
__packed_binary_op(pand_i32x2, int32x2_t, &)
|
||||
__packed_binary_op(pand_u32x2, uint32x2_t, &)
|
||||
__packed_binary_op(por_i8x8, int8x8_t, |)
|
||||
__packed_binary_op(por_u8x8, uint8x8_t, |)
|
||||
__packed_binary_op(por_i16x4, int16x4_t, |)
|
||||
__packed_binary_op(por_u16x4, uint16x4_t, |)
|
||||
__packed_binary_op(por_i32x2, int32x2_t, |)
|
||||
__packed_binary_op(por_u32x2, uint32x2_t, |)
|
||||
__packed_binary_op(pxor_i8x8, int8x8_t, ^)
|
||||
__packed_binary_op(pxor_u8x8, uint8x8_t, ^)
|
||||
__packed_binary_op(pxor_i16x4, int16x4_t, ^)
|
||||
__packed_binary_op(pxor_u16x4, uint16x4_t, ^)
|
||||
__packed_binary_op(pxor_i32x2, int32x2_t, ^)
|
||||
__packed_binary_op(pxor_u32x2, uint32x2_t, ^)
|
||||
__packed_unary_op(pnot_i8x8, int8x8_t, ~)
|
||||
__packed_unary_op(pnot_u8x8, uint8x8_t, ~)
|
||||
__packed_unary_op(pnot_i16x4, int16x4_t, ~)
|
||||
__packed_unary_op(pnot_u16x4, uint16x4_t, ~)
|
||||
__packed_unary_op(pnot_i32x2, int32x2_t, ~)
|
||||
__packed_unary_op(pnot_u32x2, uint32x2_t, ~)
|
||||
|
||||
/* Packed Widening Convert */
|
||||
__packed_widen_convert(pwcvt_i16x4, int16x4_t, int8x4_t)
|
||||
__packed_widen_convert(pwcvt_i32x2, int32x2_t, int16x2_t)
|
||||
__packed_widen_convert(pwcvtu_u16x4, uint16x4_t, uint8x4_t)
|
||||
__packed_widen_convert(pwcvtu_u32x2, uint32x2_t, uint16x2_t)
|
||||
__packed_widen_high4(pwcvth_i16x4, int16x4_t, int8x4_t)
|
||||
__packed_widen_high4(pwcvth_u16x4, uint16x4_t, uint8x4_t)
|
||||
__packed_widen_high2(pwcvth_i32x2, int32x2_t, int16x2_t)
|
||||
__packed_widen_high2(pwcvth_u32x2, uint32x2_t, uint16x2_t)
|
||||
|
||||
/* Packed Reverse (32-bit) */
|
||||
__packed_reverse4(prev_i8x4, int8x4_t)
|
||||
__packed_reverse4(prev_u8x4, uint8x4_t)
|
||||
__packed_reverse2(prev_i16x2, int16x2_t)
|
||||
__packed_reverse2(prev_u16x2, uint16x2_t)
|
||||
|
||||
/* Packed Reverse (64-bit) */
|
||||
__packed_reverse8(prev_i8x8, int8x8_t)
|
||||
__packed_reverse8(prev_u8x8, uint8x8_t)
|
||||
__packed_reverse4(prev_i16x4, int16x4_t)
|
||||
__packed_reverse4(prev_u16x4, uint16x4_t)
|
||||
__packed_reverse2(prev_i32x2, int32x2_t)
|
||||
__packed_reverse2(prev_u32x2, uint32x2_t)
|
||||
|
||||
/* Packed Zip */
|
||||
__packed_zip4(pzip_i8x8, int8x8_t, int8x4_t)
|
||||
__packed_zip4(pzip_u8x8, uint8x8_t, uint8x4_t)
|
||||
__packed_zip2(pzip_i16x4, int16x4_t, int16x2_t)
|
||||
__packed_zip2(pzip_u16x4, uint16x4_t, uint16x2_t)
|
||||
|
||||
/* Packed Unzip */
|
||||
__packed_unzipe4(punzipe_i8x4, int8x4_t, int8x8_t)
|
||||
__packed_unzipo4(punzipo_i8x4, int8x4_t, int8x8_t)
|
||||
__packed_unzipe4(punzipe_u8x4, uint8x4_t, uint8x8_t)
|
||||
__packed_unzipo4(punzipo_u8x4, uint8x4_t, uint8x8_t)
|
||||
__packed_unzipe2(punzipe_i16x2, int16x2_t, int16x4_t)
|
||||
__packed_unzipo2(punzipo_i16x2, int16x2_t, int16x4_t)
|
||||
__packed_unzipe2(punzipe_u16x2, uint16x2_t, uint16x4_t)
|
||||
__packed_unzipo2(punzipo_u16x2, uint16x2_t, uint16x4_t)
|
||||
|
||||
/* Packed Averaging Addition and Subtraction (32-bit) */
|
||||
__packed_binary_builtin(paadd_i8x4, int8x4_t, __builtin_riscv_paadd_i8x4)
|
||||
__packed_binary_builtin(paadd_i16x2, int16x2_t, __builtin_riscv_paadd_i16x2)
|
||||
__packed_binary_builtin(paaddu_u8x4, uint8x4_t, __builtin_riscv_paaddu_u8x4)
|
||||
__packed_binary_builtin(paaddu_u16x2, uint16x2_t, __builtin_riscv_paaddu_u16x2)
|
||||
__packed_binary_builtin(pasub_i8x4, int8x4_t, __builtin_riscv_pasub_i8x4)
|
||||
__packed_binary_builtin(pasub_i16x2, int16x2_t, __builtin_riscv_pasub_i16x2)
|
||||
__packed_binary_builtin(pasubu_u8x4, uint8x4_t, __builtin_riscv_pasubu_u8x4)
|
||||
__packed_binary_builtin(pasubu_u16x2, uint16x2_t, __builtin_riscv_pasubu_u16x2)
|
||||
|
||||
/* Packed Averaging Addition and Subtraction (64-bit) */
|
||||
__packed_binary_builtin(paadd_i8x8, int8x8_t, __builtin_riscv_paadd_i8x8)
|
||||
__packed_binary_builtin(paadd_i16x4, int16x4_t, __builtin_riscv_paadd_i16x4)
|
||||
__packed_binary_builtin(paadd_i32x2, int32x2_t, __builtin_riscv_paadd_i32x2)
|
||||
__packed_binary_builtin(paaddu_u8x8, uint8x8_t, __builtin_riscv_paaddu_u8x8)
|
||||
__packed_binary_builtin(paaddu_u16x4, uint16x4_t, __builtin_riscv_paaddu_u16x4)
|
||||
__packed_binary_builtin(paaddu_u32x2, uint32x2_t, __builtin_riscv_paaddu_u32x2)
|
||||
__packed_binary_builtin(pasub_i8x8, int8x8_t, __builtin_riscv_pasub_i8x8)
|
||||
__packed_binary_builtin(pasub_i16x4, int16x4_t, __builtin_riscv_pasub_i16x4)
|
||||
__packed_binary_builtin(pasub_i32x2, int32x2_t, __builtin_riscv_pasub_i32x2)
|
||||
__packed_binary_builtin(pasubu_u8x8, uint8x8_t, __builtin_riscv_pasubu_u8x8)
|
||||
__packed_binary_builtin(pasubu_u16x4, uint16x4_t, __builtin_riscv_pasubu_u16x4)
|
||||
__packed_binary_builtin(pasubu_u32x2, uint32x2_t, __builtin_riscv_pasubu_u32x2)
|
||||
|
||||
/* Packed Absolute Value and Absolute Difference (32-bit) */
|
||||
__packed_pabs(pabs_i8x4, int8x4_t, uint8x4_t)
|
||||
__packed_pabs(pabs_i16x2, int16x2_t, uint16x2_t)
|
||||
__packed_binary_builtin_cast(pabd_i8x4, int8x4_t, uint8x4_t, __builtin_riscv_pabd_i8x4)
|
||||
__packed_binary_builtin_cast(pabd_i16x2, int16x2_t, uint16x2_t, __builtin_riscv_pabd_i16x2)
|
||||
__packed_binary_builtin_cast(pabdu_u8x4, uint8x4_t, uint8x4_t, __builtin_riscv_pabdu_u8x4)
|
||||
__packed_binary_builtin_cast(pabdu_u16x2, uint16x2_t, uint16x2_t, __builtin_riscv_pabdu_u16x2)
|
||||
|
||||
/* Packed Absolute Value and Absolute Difference (64-bit) */
|
||||
__packed_pabs(pabs_i8x8, int8x8_t, uint8x8_t)
|
||||
__packed_pabs(pabs_i16x4, int16x4_t, uint16x4_t)
|
||||
__packed_binary_builtin_cast(pabd_i8x8, int8x8_t, uint8x8_t, __builtin_riscv_pabd_i8x8)
|
||||
__packed_binary_builtin_cast(pabd_i16x4, int16x4_t, uint16x4_t, __builtin_riscv_pabd_i16x4)
|
||||
__packed_binary_builtin_cast(pabdu_u8x8, uint8x8_t, uint8x8_t, __builtin_riscv_pabdu_u8x8)
|
||||
__packed_binary_builtin_cast(pabdu_u16x4, uint16x4_t, uint16x4_t, __builtin_riscv_pabdu_u16x4)
|
||||
|
||||
/* Packed Reduction Sum (32-bit) */
|
||||
__packed_reduction(predsum_i8x4_i32, int32_t, int8x4_t, __builtin_riscv_predsum_i8x4_i32)
|
||||
__packed_reduction(predsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_predsumu_u8x4_u32)
|
||||
__packed_reduction(predsum_i16x2_i32, int32_t, int16x2_t, __builtin_riscv_predsum_i16x2_i32)
|
||||
__packed_reduction(predsumu_u16x2_u32, uint32_t, uint16x2_t, __builtin_riscv_predsumu_u16x2_u32)
|
||||
|
||||
/* Packed Reduction Sum (64-bit) */
|
||||
__packed_reduction(predsum_i8x8_i32, int32_t, int8x8_t, __builtin_riscv_predsum_i8x8_i32)
|
||||
__packed_reduction(predsumu_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_predsumu_u8x8_u32)
|
||||
__packed_reduction(predsum_i16x4_i32, int32_t, int16x4_t, __builtin_riscv_predsum_i16x4_i32)
|
||||
__packed_reduction(predsumu_u16x4_u32, uint32_t, uint16x4_t, __builtin_riscv_predsumu_u16x4_u32)
|
||||
__packed_reduction(predsum_i8x8_i64, int64_t, int8x8_t, __builtin_riscv_predsum_i8x8_i64)
|
||||
__packed_reduction(predsumu_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_predsumu_u8x8_u64)
|
||||
__packed_reduction(predsum_i16x4_i64, int64_t, int16x4_t, __builtin_riscv_predsum_i16x4_i64)
|
||||
__packed_reduction(predsumu_u16x4_u64, uint64_t, uint16x4_t, __builtin_riscv_predsumu_u16x4_u64)
|
||||
__packed_reduction(predsum_i32x2_i64, int64_t, int32x2_t, __builtin_riscv_predsum_i32x2_i64)
|
||||
__packed_reduction(predsumu_u32x2_u64, uint64_t, uint32x2_t, __builtin_riscv_predsumu_u32x2_u64)
|
||||
|
||||
/* Packed Merge (32-bit) */
|
||||
__packed_merge_builtin(pmerge_u8x4, uint8x4_t, uint8x4_t, __builtin_riscv_pmerge_u8x4)
|
||||
__packed_merge_builtin(pmerge_i8x4, int8x4_t, uint8x4_t, __builtin_riscv_pmerge_i8x4)
|
||||
__packed_merge_builtin(pmerge_u16x2, uint16x2_t, uint16x2_t, __builtin_riscv_pmerge_u16x2)
|
||||
__packed_merge_builtin(pmerge_i16x2, int16x2_t, uint16x2_t, __builtin_riscv_pmerge_i16x2)
|
||||
|
||||
/* Packed Merge (64-bit) */
|
||||
__packed_merge_builtin(pmerge_u8x8, uint8x8_t, uint8x8_t, __builtin_riscv_pmerge_u8x8)
|
||||
__packed_merge_builtin(pmerge_i8x8, int8x8_t, uint8x8_t, __builtin_riscv_pmerge_i8x8)
|
||||
__packed_merge_builtin(pmerge_u16x4, uint16x4_t, uint16x4_t, __builtin_riscv_pmerge_u16x4)
|
||||
__packed_merge_builtin(pmerge_i16x4, int16x4_t, uint16x4_t, __builtin_riscv_pmerge_i16x4)
|
||||
__packed_merge_builtin(pmerge_u32x2, uint32x2_t, uint32x2_t, __builtin_riscv_pmerge_u32x2)
|
||||
__packed_merge_builtin(pmerge_i32x2, int32x2_t, uint32x2_t, __builtin_riscv_pmerge_i32x2)
|
||||
|
||||
/* Packed Absolute Difference Sum (32-bit) */
|
||||
__packed_abdsum(pabdsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumu_u8x4_u32)
|
||||
__packed_abdsum_acc(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32)
|
||||
|
||||
/* Packed Absolute Difference Sum (64-bit) */
|
||||
__packed_abdsum(pabdsumu_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumu_u8x8_u32)
|
||||
__packed_abdsum(pabdsumu_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumu_u8x8_u64)
|
||||
__packed_abdsum_acc(pabdsumau_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u32)
|
||||
__packed_abdsum_acc(pabdsumau_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u64)
|
||||
|
||||
/* Packed Saturating Absolute Value (32-bit) */
|
||||
__packed_psabs(psabs_i8x4, int8x4_t, __builtin_riscv_psabs_i8x4)
|
||||
__packed_psabs(psabs_i16x2, int16x2_t, __builtin_riscv_psabs_i16x2)
|
||||
|
||||
/* Packed Saturating Absolute Value (64-bit) */
|
||||
__packed_psabs(psabs_i8x8, int8x8_t, __builtin_riscv_psabs_i8x8)
|
||||
__packed_psabs(psabs_i16x4, int16x4_t, __builtin_riscv_psabs_i16x4)
|
||||
|
||||
// clang-format on
|
||||
|
||||
#undef __packed_splat2
|
||||
#undef __packed_splat4
|
||||
#undef __packed_splat8
|
||||
#undef __packed_splat
|
||||
#undef __packed_shift
|
||||
#undef __packed_shift8
|
||||
#undef __packed_shift16
|
||||
#undef __packed_shift32
|
||||
#undef __packed_scalar_binary_op
|
||||
#undef __packed_binary_op
|
||||
#undef __packed_unary_op
|
||||
#undef __packed_binary_builtin
|
||||
#undef __packed_sh1add
|
||||
#undef __packed_sh1sadd
|
||||
#undef __packed_cmp
|
||||
#undef __packed_pabs
|
||||
#undef __packed_binary_builtin_cast
|
||||
#undef __packed_reduction
|
||||
#undef __packed_merge_builtin
|
||||
#undef __packed_psabs
|
||||
#undef __packed_widen_convert
|
||||
#undef __packed_widen_high2
|
||||
#undef __packed_widen_high4
|
||||
#undef __packed_reverse2
|
||||
#undef __packed_reverse4
|
||||
#undef __packed_reverse8
|
||||
#undef __packed_zip2
|
||||
#undef __packed_zip4
|
||||
#undef __packed_unzipe2
|
||||
#undef __packed_unzipe4
|
||||
#undef __packed_unzipo2
|
||||
#undef __packed_unzipo4
|
||||
#undef __packed_abdsum
|
||||
#undef __packed_abdsum_acc
|
||||
#undef __DEFAULT_FN_ATTRS
|
||||
|
||||
#if defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* __RISCV_PACKED_SIMD_H */
|
||||
Vendored
+43
@@ -20,6 +20,35 @@ extern "C" {
|
||||
|
||||
#pragma clang riscv intrinsic vector
|
||||
|
||||
#define __riscv_intrinsic_v 1
|
||||
#define __riscv_intrinsic_zvabd 1
|
||||
#define __riscv_intrinsic_zvbb 1
|
||||
#define __riscv_intrinsic_zvbc 1
|
||||
#define __riscv_intrinsic_zvdot4a8i 1
|
||||
#define __riscv_intrinsic_zve32f 1
|
||||
#define __riscv_intrinsic_zve32x 1
|
||||
#define __riscv_intrinsic_zve64d 1
|
||||
#define __riscv_intrinsic_zve64f 1
|
||||
#define __riscv_intrinsic_zve64x 1
|
||||
#define __riscv_intrinsic_zvfbfa 1
|
||||
#define __riscv_intrinsic_zvfbfmin 1
|
||||
#define __riscv_intrinsic_zvfbfwma 1
|
||||
#define __riscv_intrinsic_zvfh 1
|
||||
#define __riscv_intrinsic_zvfhmin 1
|
||||
#define __riscv_intrinsic_zvfofp8min 1
|
||||
#define __riscv_intrinsic_zvkb 1
|
||||
#define __riscv_intrinsic_zvkg 1
|
||||
#define __riscv_intrinsic_zvkn 1
|
||||
#define __riscv_intrinsic_zvknc 1
|
||||
#define __riscv_intrinsic_zvkned 1
|
||||
#define __riscv_intrinsic_zvkng 1
|
||||
#define __riscv_intrinsic_zvknha 1
|
||||
#define __riscv_intrinsic_zvknhb 1
|
||||
#define __riscv_intrinsic_zvks 1
|
||||
#define __riscv_intrinsic_zvksc 1
|
||||
#define __riscv_intrinsic_zvksed 1
|
||||
#define __riscv_intrinsic_zvksg 1
|
||||
#define __riscv_intrinsic_zvksh 1
|
||||
|
||||
enum __RISCV_FRM {
|
||||
__RISCV_FRM_RNE = 0,
|
||||
@@ -415,6 +444,20 @@ typedef __rvv_bfloat16m2x4_t vbfloat16m2x4_t;
|
||||
typedef __rvv_bfloat16m4_t vbfloat16m4_t;
|
||||
typedef __rvv_bfloat16m4x2_t vbfloat16m4x2_t;
|
||||
typedef __rvv_bfloat16m8_t vbfloat16m8_t;
|
||||
typedef __rvv_float8e4m3mf8_t vfloat8e4m3mf8_t;
|
||||
typedef __rvv_float8e4m3mf4_t vfloat8e4m3mf4_t;
|
||||
typedef __rvv_float8e4m3mf2_t vfloat8e4m3mf2_t;
|
||||
typedef __rvv_float8e4m3m1_t vfloat8e4m3m1_t;
|
||||
typedef __rvv_float8e4m3m2_t vfloat8e4m3m2_t;
|
||||
typedef __rvv_float8e4m3m4_t vfloat8e4m3m4_t;
|
||||
typedef __rvv_float8e4m3m8_t vfloat8e4m3m8_t;
|
||||
typedef __rvv_float8e5m2mf8_t vfloat8e5m2mf8_t;
|
||||
typedef __rvv_float8e5m2mf4_t vfloat8e5m2mf4_t;
|
||||
typedef __rvv_float8e5m2mf2_t vfloat8e5m2mf2_t;
|
||||
typedef __rvv_float8e5m2m1_t vfloat8e5m2m1_t;
|
||||
typedef __rvv_float8e5m2m2_t vfloat8e5m2m2_t;
|
||||
typedef __rvv_float8e5m2m4_t vfloat8e5m2m4_t;
|
||||
typedef __rvv_float8e5m2m8_t vfloat8e5m2m8_t;
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
||||
Vendored
+18
@@ -13,6 +13,24 @@
|
||||
|
||||
#pragma clang riscv intrinsic sifive_vector
|
||||
|
||||
#define __riscv_intrinsic_xsfmm32a16f 1
|
||||
#define __riscv_intrinsic_xsfmm32a32f 1
|
||||
#define __riscv_intrinsic_xsfmm32a8f 1
|
||||
#define __riscv_intrinsic_xsfmm32a8i 1
|
||||
#define __riscv_intrinsic_xsfmm32a 1
|
||||
#define __riscv_intrinsic_xsfmm64a64f 1
|
||||
#define __riscv_intrinsic_xsfmmbase 1
|
||||
#define __riscv_intrinsic_xsfvcp 1
|
||||
#define __riscv_intrinsic_xsfvfbfexp16e 1
|
||||
#define __riscv_intrinsic_xsfvfexp16e 1
|
||||
#define __riscv_intrinsic_xsfvfexp32e 1
|
||||
#define __riscv_intrinsic_xsfvfexpa 1
|
||||
#define __riscv_intrinsic_xsfvfexpa64e 1
|
||||
#define __riscv_intrinsic_xsfvfnrclipxfqf 1
|
||||
#define __riscv_intrinsic_xsfvfwmaccqqq 1
|
||||
#define __riscv_intrinsic_xsfvqmaccdod 1
|
||||
#define __riscv_intrinsic_xsfvqmaccqoq 1
|
||||
|
||||
#define __riscv_sf_vc_x_se_u8mf4(p27_26, p24_20, p11_7, rs1, vl) \
|
||||
__riscv_sf_vc_x_se(p27_26, p24_20, p11_7, (uint8_t)rs1, 8, 6, vl)
|
||||
#define __riscv_sf_vc_x_se_u8mf2(p27_26, p24_20, p11_7, rs1, vl) \
|
||||
|
||||
Vendored
+2
-29
@@ -27,9 +27,6 @@ _Pragma("omp begin declare variant match(device = {arch(spirv64)})");
|
||||
#define __gpu_global __attribute__((address_space(1)))
|
||||
#define __gpu_generic __attribute__((address_space(4)))
|
||||
|
||||
// Attribute to declare a function as a kernel.
|
||||
#define __gpu_kernel __attribute__((device_kernel, visibility("protected")))
|
||||
|
||||
// Returns the number of workgroups in the 'x' dimension of the grid.
|
||||
_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_blocks_x(void) {
|
||||
return __builtin_spirv_num_workgroups(0);
|
||||
@@ -146,37 +143,13 @@ __gpu_shuffle_idx_u32(uint64_t __lane_mask, uint32_t __idx, uint32_t __x,
|
||||
return __builtin_spirv_subgroup_shuffle(__x, __lane);
|
||||
}
|
||||
|
||||
// Returns a bitmask marking all lanes that have the same value of __x.
|
||||
_DEFAULT_FN_ATTRS static __inline__ uint64_t
|
||||
__gpu_match_any_u32(uint64_t __lane_mask, uint32_t __x) {
|
||||
return __gpu_match_any_u32_impl(__lane_mask, __x);
|
||||
}
|
||||
|
||||
// Returns a bitmask marking all lanes that have the same value of __x.
|
||||
_DEFAULT_FN_ATTRS static __inline__ uint64_t
|
||||
__gpu_match_any_u64(uint64_t __lane_mask, uint64_t __x) {
|
||||
return __gpu_match_any_u64_impl(__lane_mask, __x);
|
||||
}
|
||||
|
||||
// Returns the current lane mask if every lane contains __x.
|
||||
_DEFAULT_FN_ATTRS static __inline__ uint64_t
|
||||
__gpu_match_all_u32(uint64_t __lane_mask, uint32_t __x) {
|
||||
return __gpu_match_all_u32_impl(__lane_mask, __x);
|
||||
}
|
||||
|
||||
// Returns the current lane mask if every lane contains __x.
|
||||
_DEFAULT_FN_ATTRS static __inline__ uint64_t
|
||||
__gpu_match_all_u64(uint64_t __lane_mask, uint64_t __x) {
|
||||
return __gpu_match_all_u64_impl(__lane_mask, __x);
|
||||
}
|
||||
|
||||
// SPIR-V does not expose this, always return false.
|
||||
_DEFAULT_FN_ATTRS static __inline__ bool __gpu_is_ptr_local(void *ptr) {
|
||||
_DEFAULT_FN_ATTRS static __inline__ bool __gpu_is_ptr_local(const void *ptr) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// SPIR-V does not expose this, always return false.
|
||||
_DEFAULT_FN_ATTRS static __inline__ bool __gpu_is_ptr_private(void *ptr) {
|
||||
_DEFAULT_FN_ATTRS static __inline__ bool __gpu_is_ptr_private(const void *ptr) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
Vendored
+5
-14
@@ -805,25 +805,16 @@ typedef __UINTMAX_TYPE__ uintmax_t;
|
||||
#endif
|
||||
|
||||
/* C99 7.18.3 Limits of other integer types. */
|
||||
#define SIG_ATOMIC_MIN __INTN_MIN(__SIG_ATOMIC_WIDTH__)
|
||||
#define SIG_ATOMIC_MAX __INTN_MAX(__SIG_ATOMIC_WIDTH__)
|
||||
#ifdef __WINT_UNSIGNED__
|
||||
# define WINT_MIN __UINTN_C(__WINT_WIDTH__, 0)
|
||||
# define WINT_MAX __UINTN_MAX(__WINT_WIDTH__)
|
||||
#else
|
||||
# define WINT_MIN __INTN_MIN(__WINT_WIDTH__)
|
||||
# define WINT_MAX __INTN_MAX(__WINT_WIDTH__)
|
||||
#endif
|
||||
#define SIG_ATOMIC_MIN __SIG_ATOMIC_MIN__
|
||||
#define SIG_ATOMIC_MAX __SIG_ATOMIC_MAX__
|
||||
#define WINT_MIN __WINT_MIN__
|
||||
#define WINT_MAX __WINT_MAX__
|
||||
|
||||
#ifndef WCHAR_MAX
|
||||
# define WCHAR_MAX __WCHAR_MAX__
|
||||
#endif
|
||||
#ifndef WCHAR_MIN
|
||||
# if __WCHAR_MAX__ == __INTN_MAX(__WCHAR_WIDTH__)
|
||||
# define WCHAR_MIN __INTN_MIN(__WCHAR_WIDTH__)
|
||||
# else
|
||||
# define WCHAR_MIN __UINTN_C(__WCHAR_WIDTH__, 0)
|
||||
# endif
|
||||
#define WCHAR_MIN __WCHAR_MIN__
|
||||
#endif
|
||||
|
||||
/* 7.18.4.2 Macros for greatest-width integer constants. */
|
||||
|
||||
Vendored
+1
-1
@@ -207,7 +207,7 @@ vec_insert(unsigned long long __scalar, __vector unsigned long long __vec,
|
||||
#if __ARCH__ >= 12
|
||||
static inline __ATTRS_o_ai __vector float
|
||||
vec_insert(float __scalar, __vector float __vec, int __index) {
|
||||
__vec[__index & 1] = __scalar;
|
||||
__vec[__index & 3] = __scalar;
|
||||
return __vec;
|
||||
}
|
||||
#endif
|
||||
|
||||
Vendored
+17
@@ -45,6 +45,7 @@ typedef int __i32x2 __attribute__((__vector_size__(8), __aligned__(8)));
|
||||
typedef unsigned int __u32x2
|
||||
__attribute__((__vector_size__(8), __aligned__(8)));
|
||||
typedef float __f32x2 __attribute__((__vector_size__(8), __aligned__(8)));
|
||||
typedef __fp16 __f16x4 __attribute__((__vector_size__(8), __aligned__(8)));
|
||||
|
||||
#define __DEFAULT_FN_ATTRS \
|
||||
__attribute__((__always_inline__, __nodebug__, __target__("simd128"), \
|
||||
@@ -2010,6 +2011,22 @@ static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_convert_u16x8(v128_t __a) {
|
||||
return (v128_t) __builtin_convertvector((__u16x8)__a, __f16x8);
|
||||
}
|
||||
|
||||
static __inline__ v128_t __FP16_FN_ATTRS
|
||||
wasm_f32x4_promote_low_f16x8(v128_t __a) {
|
||||
return (v128_t) __builtin_convertvector(
|
||||
(__f16x4){((__f16x8)__a)[0], ((__f16x8)__a)[1], ((__f16x8)__a)[2],
|
||||
((__f16x8)__a)[3]},
|
||||
__f32x4);
|
||||
}
|
||||
|
||||
static __inline__ v128_t __FP16_FN_ATTRS
|
||||
wasm_f16x8_demote_f32x4_zero(v128_t __a) {
|
||||
return (v128_t) __builtin_convertvector(
|
||||
__builtin_shufflevector((__f32x4)__a, (__f32x4){0, 0, 0, 0}, 0, 1, 2, 3,
|
||||
4, 5, 6, 7),
|
||||
__f16x8);
|
||||
}
|
||||
|
||||
static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_relaxed_madd(v128_t __a,
|
||||
v128_t __b,
|
||||
v128_t __c) {
|
||||
|
||||
Vendored
+8
-7
@@ -341,7 +341,8 @@ _mm_rsqrt_ps(__m128 __a)
|
||||
/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the
|
||||
/// minimum value between both operands. The upper 96 bits are copied from
|
||||
/// the upper 96 bits of the first source operand.
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_min_ss(__m128 __a, __m128 __b) {
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_min_ss(__m128 __a,
|
||||
__m128 __b) {
|
||||
return __builtin_ia32_minss((__v4sf)__a, (__v4sf)__b);
|
||||
}
|
||||
|
||||
@@ -384,7 +385,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_min_ps(__m128 __a,
|
||||
/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the
|
||||
/// maximum value between both operands. The upper 96 bits are copied from
|
||||
/// the upper 96 bits of the first source operand.
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_max_ss(__m128 __a, __m128 __b) {
|
||||
static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_max_ss(__m128 __a,
|
||||
__m128 __b) {
|
||||
return __builtin_ia32_maxss((__v4sf)__a, (__v4sf)__b);
|
||||
}
|
||||
|
||||
@@ -2569,11 +2571,10 @@ _mm_avg_pu16(__m64 __a, __m64 __b) {
|
||||
/// \returns A 64-bit integer vector whose lower 16 bits contain the sums of the
|
||||
/// sets of absolute differences between both operands. The upper bits are
|
||||
/// cleared.
|
||||
static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
|
||||
_mm_sad_pu8(__m64 __a, __m64 __b)
|
||||
{
|
||||
return __trunc64(__builtin_ia32_psadbw128((__v16qi)__zext128(__a),
|
||||
(__v16qi)__zext128(__b)));
|
||||
static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
|
||||
_mm_sad_pu8(__m64 __a, __m64 __b) {
|
||||
return __trunc64(__builtin_ia32_psadbw128((__v16qu)__zext128(__a),
|
||||
(__v16qu)__zext128(__b)));
|
||||
}
|
||||
|
||||
#if defined(__cplusplus)
|
||||
|
||||
Reference in new issue
Block a user