elee1766 proved this is a worthwhile tradeoff to make on amd64 in CL 796641. Pretty much any CPU we care about has a similar or better performance for left shift by a constant. So optimize the accumulation chains on all arches. Cq-Include-Trybots: luci.golang.try:gotip-linux-ppc64_power10,gotip-linux-ppc64le_power10,gotip-linux-ppc64le_power9 Change-Id: I25004d295a5ea59d8eda736318e04244e95daabe Reviewed-on: https://go-review.googlesource.com/c/go/+/809400 Auto-Submit: Jorropo <jorropo.pgm@gmail.com> Reviewed-by: Keith Randall <khr@golang.org> Reviewed-by: Keith Randall <khr@google.com> Reviewed-by: Dmitri Shuralyov <dmitshur@google.com> LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
881 lines
16 KiB
Go
881 lines
16 KiB
Go
// asmcheck
|
|
|
|
// Copyright 2018 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package codegen
|
|
|
|
import (
|
|
"crypto/subtle"
|
|
"math/bits"
|
|
)
|
|
|
|
func cmovint(c int) int {
|
|
x := c + 4
|
|
if x < 0 {
|
|
x = 182
|
|
}
|
|
// amd64:"CMOVQLT"
|
|
// arm64:"CSEL LT"
|
|
// ppc64x:"ISEL [$]0"
|
|
// wasm:"Select"
|
|
return x
|
|
}
|
|
|
|
func cmovchan(x, y chan int) chan int {
|
|
if x != y {
|
|
x = y
|
|
}
|
|
// amd64:"CMOVQNE"
|
|
// arm64:"CSEL NE"
|
|
// ppc64x:"ISEL [$]2"
|
|
// wasm:"Select"
|
|
return x
|
|
}
|
|
|
|
func cmovuintptr(x, y uintptr) uintptr {
|
|
if x < y {
|
|
x = -y
|
|
}
|
|
// amd64:"CMOVQ(HI|CS)"
|
|
// arm64:"CSNEG LS"
|
|
// ppc64x:"ISEL [$]1"
|
|
// wasm:"Select"
|
|
return x
|
|
}
|
|
|
|
func cmov32bit(x, y uint32) uint32 {
|
|
if x < y {
|
|
x = -y
|
|
}
|
|
// amd64:"CMOVL(HI|CS)"
|
|
// arm64:"CSNEG (LS|HS)"
|
|
// ppc64x:"ISEL [$]1"
|
|
// wasm:"Select"
|
|
return x
|
|
}
|
|
|
|
func cmov16bit(x, y uint16) uint16 {
|
|
if x < y {
|
|
x = -y
|
|
}
|
|
// amd64:"CMOVW(HI|CS)"
|
|
// arm64:"CSNEG (LS|HS)"
|
|
// ppc64x:"ISEL [$][01]"
|
|
// wasm:"Select"
|
|
return x
|
|
}
|
|
|
|
// Floating point comparison. For EQ/NE, we must
|
|
// generate special code to handle NaNs.
|
|
func cmovfloateq(x, y float64) int {
|
|
a := 128
|
|
if x == y {
|
|
a = 256
|
|
}
|
|
// amd64:"CMOVQNE" "CMOVQPC"
|
|
// arm64:"CSEL EQ"
|
|
// ppc64x:"ISEL [$]2"
|
|
// wasm:"Select"
|
|
return a
|
|
}
|
|
|
|
func cmovfloatne(x, y float64) int {
|
|
a := 128
|
|
if x != y {
|
|
a = 256
|
|
}
|
|
// amd64:"CMOVQNE" "CMOVQPS"
|
|
// arm64:"CSEL NE"
|
|
// ppc64x:"ISEL [$]2"
|
|
// wasm:"Select"
|
|
return a
|
|
}
|
|
|
|
//go:noinline
|
|
func frexp(f float64) (frac float64, exp int) {
|
|
return 1.0, 4
|
|
}
|
|
|
|
//go:noinline
|
|
func ldexp(frac float64, exp int) float64 {
|
|
return 1.0
|
|
}
|
|
|
|
// Generate a CMOV with a floating comparison and integer move.
|
|
func cmovfloatint2(x, y float64) float64 {
|
|
yfr, yexp := 4.0, 5
|
|
|
|
r := x
|
|
for r >= y {
|
|
rfr, rexp := frexp(r)
|
|
if rfr < yfr {
|
|
rexp = rexp - 42
|
|
}
|
|
// amd64:"CMOVQHI"
|
|
// arm64:"CSEL MI"
|
|
// ppc64x:"ISEL [$]0"
|
|
// wasm:"Select"
|
|
r = r - ldexp(y, rexp-yexp)
|
|
}
|
|
return r
|
|
}
|
|
|
|
func cmovloaded(x [4]int, y int) int {
|
|
if x[2] != 0 {
|
|
y = x[2]
|
|
} else {
|
|
y = y >> 2
|
|
}
|
|
// amd64:"CMOVQNE"
|
|
// arm64:"CSEL NE"
|
|
// ppc64x:"ISEL [$]2"
|
|
// wasm:"Select"
|
|
return y
|
|
}
|
|
|
|
func cmovuintptr2(x, y uintptr) uintptr {
|
|
a := x * 2
|
|
if a == 0 {
|
|
a = 256
|
|
}
|
|
// amd64:"CMOVQEQ"
|
|
// arm64:"CSEL EQ"
|
|
// ppc64x:"ISEL [$]2"
|
|
// wasm:"Select"
|
|
return a
|
|
}
|
|
|
|
// Floating point CMOVs are not supported by amd64/arm64/ppc64x
|
|
func cmovfloatmove(x, y int) float64 {
|
|
a := 1.0
|
|
if x <= y {
|
|
a = 2.0
|
|
}
|
|
// amd64:-"CMOV"
|
|
// arm64:-"CSEL"
|
|
// ppc64x:-"ISEL"
|
|
// wasm:-"Select"
|
|
return a
|
|
}
|
|
|
|
// On amd64, the following patterns trigger comparison inversion.
|
|
// Test that we correctly invert the CMOV condition
|
|
var gsink int64
|
|
var gusink uint64
|
|
|
|
func cmovinvert1(x, y int64) int64 {
|
|
if x < gsink {
|
|
y = -y
|
|
}
|
|
// amd64:"CMOVQGT"
|
|
return y
|
|
}
|
|
func cmovinvert2(x, y int64) int64 {
|
|
if x <= gsink {
|
|
y = -y
|
|
}
|
|
// amd64:"CMOVQGE"
|
|
return y
|
|
}
|
|
func cmovinvert3(x, y int64) int64 {
|
|
if x == gsink {
|
|
y = -y
|
|
}
|
|
// amd64:"CMOVQEQ"
|
|
return y
|
|
}
|
|
func cmovinvert4(x, y int64) int64 {
|
|
if x != gsink {
|
|
y = -y
|
|
}
|
|
// amd64:"CMOVQNE"
|
|
return y
|
|
}
|
|
func cmovinvert5(x, y uint64) uint64 {
|
|
if x > gusink {
|
|
y = -y
|
|
}
|
|
// amd64:"CMOVQCS"
|
|
return y
|
|
}
|
|
func cmovinvert6(x, y uint64) uint64 {
|
|
if x >= gusink {
|
|
y = -y
|
|
}
|
|
// amd64:"CMOVQLS"
|
|
return y
|
|
}
|
|
|
|
func cmovload(a []int, i int, b bool) int {
|
|
if b {
|
|
i += 42
|
|
}
|
|
// See issue 26306
|
|
// amd64:-"CMOVQNE"
|
|
return a[i]
|
|
}
|
|
|
|
func cmovstore(a []int, i int, b bool) {
|
|
if b {
|
|
i += 42
|
|
}
|
|
// amd64:"CMOVQNE"
|
|
a[i] = 7
|
|
}
|
|
|
|
var r0, r1, r2, r3, r4, r5 int
|
|
|
|
func cmovinc(cond bool, a, b, c int) {
|
|
var x0, x1 int
|
|
|
|
if cond {
|
|
x0 = a
|
|
} else {
|
|
x0 = b + 1
|
|
}
|
|
// arm64:"CSINC NE" -"CSEL"
|
|
r0 = x0
|
|
|
|
if cond {
|
|
x1 = b + 1
|
|
} else {
|
|
x1 = a
|
|
}
|
|
// arm64:"CSINC EQ" -"CSEL"
|
|
r1 = x1
|
|
|
|
if cond {
|
|
c++
|
|
}
|
|
// arm64:"CSINC EQ" -"CSEL"
|
|
r2 = c
|
|
}
|
|
|
|
func cmovinv(cond bool, a, b int) {
|
|
var x0, x1 int
|
|
|
|
if cond {
|
|
x0 = a
|
|
} else {
|
|
x0 = ^b
|
|
}
|
|
// arm64:"CSINV NE" -"CSEL"
|
|
r0 = x0
|
|
|
|
if cond {
|
|
x1 = ^b
|
|
} else {
|
|
x1 = a
|
|
}
|
|
// arm64:"CSINV EQ" -"CSEL"
|
|
r1 = x1
|
|
}
|
|
|
|
func cmovneg(cond bool, a, b, c int) {
|
|
var x0, x1 int
|
|
|
|
if cond {
|
|
x0 = a
|
|
} else {
|
|
x0 = -b
|
|
}
|
|
// arm64:"CSNEG NE" -"CSEL"
|
|
r0 = x0
|
|
|
|
if cond {
|
|
x1 = -b
|
|
} else {
|
|
x1 = a
|
|
}
|
|
// arm64:"CSNEG EQ" -"CSEL"
|
|
r1 = x1
|
|
}
|
|
|
|
func cmovsetm(cond bool, x int) {
|
|
var x0, x1 int
|
|
|
|
if cond {
|
|
x0 = -1
|
|
} else {
|
|
x0 = 0
|
|
}
|
|
// arm64:"CSETM NE" -"CSEL"
|
|
r0 = x0
|
|
|
|
if cond {
|
|
x1 = 0
|
|
} else {
|
|
x1 = -1
|
|
}
|
|
// arm64:"CSETM EQ" -"CSEL"
|
|
r1 = x1
|
|
}
|
|
|
|
func cmovFcmp0(s, t float64, a, b int) {
|
|
var x0, x1, x2, x3, x4, x5 int
|
|
|
|
if s < t {
|
|
x0 = a
|
|
} else {
|
|
x0 = b + 1
|
|
}
|
|
// arm64:"CSINC MI" -"CSEL"
|
|
r0 = x0
|
|
|
|
if s <= t {
|
|
x1 = a
|
|
} else {
|
|
x1 = ^b
|
|
}
|
|
// arm64:"CSINV LS" -"CSEL"
|
|
r1 = x1
|
|
|
|
if s > t {
|
|
x2 = a
|
|
} else {
|
|
x2 = -b
|
|
}
|
|
// arm64:"CSNEG MI" -"CSEL"
|
|
r2 = x2
|
|
|
|
if s >= t {
|
|
x3 = -1
|
|
} else {
|
|
x3 = 0
|
|
}
|
|
// arm64:"CSETM LS" -"CSEL"
|
|
r3 = x3
|
|
|
|
if s == t {
|
|
x4 = a
|
|
} else {
|
|
x4 = b + 1
|
|
}
|
|
// arm64:"CSINC EQ" -"CSEL"
|
|
r4 = x4
|
|
|
|
if s != t {
|
|
x5 = a
|
|
} else {
|
|
x5 = b + 1
|
|
}
|
|
// arm64:"CSINC NE" -"CSEL"
|
|
r5 = x5
|
|
}
|
|
|
|
func cmovFcmp1(s, t float64, a, b int) {
|
|
var x0, x1, x2, x3, x4, x5 int
|
|
|
|
if s < t {
|
|
x0 = b + 1
|
|
} else {
|
|
x0 = a
|
|
}
|
|
// arm64:"CSINC PL" -"CSEL"
|
|
r0 = x0
|
|
|
|
if s <= t {
|
|
x1 = ^b
|
|
} else {
|
|
x1 = a
|
|
}
|
|
// arm64:"CSINV HI" -"CSEL"
|
|
r1 = x1
|
|
|
|
if s > t {
|
|
x2 = -b
|
|
} else {
|
|
x2 = a
|
|
}
|
|
// arm64:"CSNEG PL" -"CSEL"
|
|
r2 = x2
|
|
|
|
if s >= t {
|
|
x3 = 0
|
|
} else {
|
|
x3 = -1
|
|
}
|
|
// arm64:"CSETM HI" -"CSEL"
|
|
r3 = x3
|
|
|
|
if s == t {
|
|
x4 = b + 1
|
|
} else {
|
|
x4 = a
|
|
}
|
|
// arm64:"CSINC NE" -"CSEL"
|
|
r4 = x4
|
|
|
|
if s != t {
|
|
x5 = b + 1
|
|
} else {
|
|
x5 = a
|
|
}
|
|
// arm64:"CSINC EQ" -"CSEL"
|
|
r5 = x5
|
|
}
|
|
|
|
func cmovzero1(c bool) int {
|
|
var x int
|
|
if c {
|
|
x = 182
|
|
}
|
|
// loong64:"MASKEQZ" -"MASKNEZ"
|
|
return x
|
|
}
|
|
|
|
func cmovzero2(c bool) int {
|
|
var x int
|
|
if !c {
|
|
x = 182
|
|
}
|
|
// loong64:"MASKNEZ" -"MASKEQZ"
|
|
return x
|
|
}
|
|
|
|
// Conditionally selecting between a value or 0 can be done without
|
|
// an extra load of 0 to a register on PPC64 by using R0 (which always
|
|
// holds the value $0) instead. Verify both cases where either arg1
|
|
// or arg2 is zero.
|
|
func cmovzeroreg0(a, b int) int {
|
|
x := 0
|
|
if a == b {
|
|
x = a
|
|
}
|
|
// ppc64x:"ISEL [$]2, R[0-9]+, R0, R[0-9]+"
|
|
return x
|
|
}
|
|
|
|
func cmovzeroreg1(a, b int) int {
|
|
x := a
|
|
if a == b {
|
|
x = 0
|
|
}
|
|
// ppc64x:"ISEL [$]2, R0, R[0-9]+, R[0-9]+"
|
|
return x
|
|
}
|
|
|
|
func cmovmathadd(a uint, b bool) uint {
|
|
if b {
|
|
a++
|
|
}
|
|
// amd64:"ADDQ" -"CMOV"
|
|
// arm64:"CSINC" -"CSEL"
|
|
// ppc64x:"ADD" -"ISEL"
|
|
// wasm:"I64Add" -"Select"
|
|
return a
|
|
}
|
|
func cmovmathaddelse(a uint, b bool) uint {
|
|
if !b {
|
|
a++
|
|
}
|
|
// amd64:"ADDQ" -"CMOV"
|
|
// arm64:"CSINC" -"CSEL"
|
|
// ppc64x:"ADD" -"ISEL"
|
|
// wasm:"I64Add" -"Select"
|
|
return a
|
|
}
|
|
|
|
func cmovmathadd2(a uint, b bool) uint {
|
|
if b {
|
|
a += 2
|
|
}
|
|
// amd64:"LEAQ" -"CMOV" -"MUL"
|
|
// arm64:"ADD R[0-9]+<<1" -"CSEL" -"MUL"
|
|
// ppc64x:"CLRLSLDI [$]56, R[0-9]+, [$]1" "ADD" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
func cmovmathadd2else(a uint, b bool) uint {
|
|
if !b {
|
|
a += 2
|
|
}
|
|
// amd64:"LEAQ" -"CMOV" -"MUL"
|
|
// arm64:"ADD R[0-9]+<<1" -"CSEL" -"MUL"
|
|
// ppc64x:"CLRLSLDI [$]56, R[0-9]+, [$]1" "ADD" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
|
|
func cmovmathadd4(a uint, b bool) uint {
|
|
if b {
|
|
a += 4
|
|
}
|
|
// amd64:"LEAQ" -"CMOV" -"MUL"
|
|
// arm64:"ADD R[0-9]+<<2" -"CSEL" -"MUL"
|
|
// ppc64x:"CLRLSLDI [$]56, R[0-9]+, [$]2" "ADD" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
func cmovmathadd4else(a uint, b bool) uint {
|
|
if !b {
|
|
a += 4
|
|
}
|
|
// amd64:"LEAQ" -"CMOV" -"MUL"
|
|
// arm64:"ADD R[0-9]+<<2" -"CSEL" -"MUL"
|
|
// ppc64x:"CLRLSLDI [$]56, R[0-9]+, [$]2" "ADD" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
|
|
func cmovmathadd8(a uint, b bool) uint {
|
|
if b {
|
|
a += 8
|
|
}
|
|
// amd64:"LEAQ" -"CMOV" -"MUL"
|
|
// arm64:"ADD R[0-9]+<<3" -"CSEL" -"MUL"
|
|
// ppc64x:"CLRLSLDI [$]56, R[0-9]+, [$]3" "ADD" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
func cmovmathadd8else(a uint, b bool) uint {
|
|
if !b {
|
|
a += 8
|
|
}
|
|
// amd64:"LEAQ" -"CMOV" -"MUL"
|
|
// arm64:"ADD R[0-9]+<<3" -"CSEL" -"MUL"
|
|
// ppc64x:"CLRLSLDI [$]56, R[0-9]+, [$]3" "ADD" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
|
|
func cmovmathadd16(a uint, b bool) uint {
|
|
if b {
|
|
a += 16
|
|
}
|
|
// amd64:"ADDQ" -"CMOV"
|
|
// arm64:"ADD R[0-9]+<<4" -"CSEL" -"MUL"
|
|
// ppc64x:"CLRLSLDI [$]56, R[0-9]+, [$]4" "ADD" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
func cmovmathadd16else(a uint, b bool) uint {
|
|
if !b {
|
|
a += 16
|
|
}
|
|
// amd64:"ADDQ" -"CMOV"
|
|
// arm64:"ADD R[0-9]+<<4" -"CSEL" -"MUL"
|
|
// ppc64x:"CLRLSLDI [$]56, R[0-9]+, [$]4" "ADD" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
|
|
func cmovmathadd9223372036854775808(a uint, b bool) uint {
|
|
if b {
|
|
a += 1 << 63
|
|
}
|
|
// amd64:"ADDQ" -"CMOV"
|
|
// arm64:"ADD R[0-9]+<<63" -"CSEL" -"MUL"
|
|
// ppc64x:"SLD [$]63" "ADD" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
func cmovmathadd9223372036854775808else(a uint, b bool) uint {
|
|
if !b {
|
|
a += 1 << 63
|
|
}
|
|
// amd64:"ADDQ" -"CMOV"
|
|
// arm64:"ADD R[0-9]+<<63" -"CSEL" -"MUL"
|
|
// ppc64x:"SLD [$]63" "ADD" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
|
|
func cmovmathsub(a uint, b bool) uint {
|
|
if b {
|
|
a--
|
|
}
|
|
// amd64:"SUBQ" -"CMOV"
|
|
// arm64:"SUB" -"CSEL"
|
|
// ppc64x:"SUB" -"ISEL"
|
|
// wasm:"I64Sub" -"Select"
|
|
return a
|
|
}
|
|
func cmovmathsubelse(a uint, b bool) uint {
|
|
if !b {
|
|
a--
|
|
}
|
|
// amd64:"SUBQ" -"CMOV"
|
|
// arm64:"SUB" -"CSEL"
|
|
// ppc64x:"SUB" -"ISEL"
|
|
// wasm:"I64Sub" -"Select"
|
|
return a
|
|
}
|
|
|
|
func cmovmathsub2(a uint, b bool) uint {
|
|
if b {
|
|
a -= 2
|
|
}
|
|
// amd64:"SUBQ" -"CMOV"
|
|
// arm64 :"SUB R[0-9]+<<1" -"CSEL" -"MUL"
|
|
// ppc64x:"CLRLSLDI [$]56, R[0-9]+, [$]1" "SUB" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
func cmovmathsub2else(a uint, b bool) uint {
|
|
if !b {
|
|
a -= 2
|
|
}
|
|
// amd64:"SUBQ" -"CMOV"
|
|
// arm64 :"SUB R[0-9]+<<1" -"CSEL" -"MUL"
|
|
// ppc64x:"CLRLSLDI [$]56, R[0-9]+, [$]1" "SUB" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
|
|
// Theses two are special because in fixed width two's complement -(1<<(size-1)) == 1<<(size-1).
|
|
// It doesn't matter if they are implemented with SUB or ADD.
|
|
func cmovmathsub9223372036854775808(a uint, b bool) uint {
|
|
if b {
|
|
a -= 1 << 63
|
|
}
|
|
// amd64:"(SUBQ|ADDQ)" -"CMOV"
|
|
// arm64:"(SUB|ADD) R[0-9]+<<63" -"CSEL" -"MUL"
|
|
// ppc64x:"SLD [$]63" "ADD" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
func cmovmathsub9223372036854775808else(a uint, b bool) uint {
|
|
if !b {
|
|
a -= 1 << 63
|
|
}
|
|
// amd64:"(SUBQ|ADDQ)" -"CMOV"
|
|
// arm64:"(SUB|ADD) R[0-9]+<<63" -"CSEL" -"MUL"
|
|
// ppc64x:"SLD [$]63" "ADD" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
|
|
func cmovmathdouble(a uint, b bool) uint {
|
|
if b {
|
|
a *= 2
|
|
}
|
|
// amd64:"SHL" -"CMOV"
|
|
// amd64/v3:"SHL" -"CMOV" -"MOV"
|
|
// arm64:"LSL" -"CSEL"
|
|
// wasm:"I64Shl" -"Select"
|
|
return a
|
|
}
|
|
func cmovmathdoubleelse(a uint, b bool) uint {
|
|
if !b {
|
|
a *= 2
|
|
}
|
|
// amd64:"SHL" -"CMOV"
|
|
// amd64/v3:"SHL" -"CMOV" -"MOV"
|
|
// arm64:"LSL" -"CSEL"
|
|
// wasm:"I64Shl" -"Select"
|
|
return a
|
|
}
|
|
|
|
func cmovmathhalvei(a int, b bool) int {
|
|
if b {
|
|
// For some reason the compiler attributes the shift to inside this block rather than where the Phi node is.
|
|
// arm64:"ASR" -"CSEL"
|
|
// wasm:"I64ShrS" -"Select"
|
|
a /= 2
|
|
}
|
|
// arm64:-"CSEL"
|
|
// wasm:-"Select"
|
|
return a
|
|
}
|
|
func cmovmathhalveielse(a int, b bool) int {
|
|
if !b {
|
|
// For some reason the compiler attributes the shift to inside this block rather than where the Phi node is.
|
|
// arm64:"ASR" -"CSEL"
|
|
// wasm:"I64ShrS" -"Select"
|
|
a /= 2
|
|
}
|
|
// arm64:-"CSEL"
|
|
// wasm:-"Select"
|
|
return a
|
|
}
|
|
|
|
func cmovmathhalveu(a uint, b bool) uint {
|
|
if b {
|
|
a /= 2
|
|
}
|
|
// amd64:"SHR" -"CMOV"
|
|
// amd64/v3:"SHR" -"CMOV" -"MOV"
|
|
// arm64:"LSR" -"CSEL"
|
|
// wasm:"I64ShrU" -"Select"
|
|
return a
|
|
}
|
|
func cmovmathhalveuelse(a uint, b bool) uint {
|
|
if !b {
|
|
a /= 2
|
|
}
|
|
// amd64:"SHR" -"CMOV"
|
|
// amd64/v3:"SHR" -"CMOV" -"MOV"
|
|
// arm64:"LSR" -"CSEL"
|
|
// wasm:"I64ShrU" -"Select"
|
|
return a
|
|
}
|
|
|
|
func cmovmathor(a uint, b bool) uint {
|
|
if b {
|
|
a |= 1
|
|
}
|
|
// amd64:"ORQ" -"CMOV"
|
|
// arm64:"ORR" -"CSEL"
|
|
// ppc64x:"OR" -"ISEL"
|
|
// wasm:"I64Or" -"Select"
|
|
return a
|
|
}
|
|
func cmovmathorelse(a uint, b bool) uint {
|
|
if !b {
|
|
a |= 1
|
|
}
|
|
// amd64:"ORQ" -"CMOV"
|
|
// arm64:"ORR" -"CSEL"
|
|
// ppc64x:"OR" -"ISEL"
|
|
// wasm:"I64Or" -"Select"
|
|
return a
|
|
}
|
|
|
|
func cmovmathor2(a uint, b bool) uint {
|
|
if b {
|
|
a |= 2
|
|
}
|
|
// amd64:"ORQ" -"CMOV"
|
|
// arm64:"ORR R[0-9]+<<1" -"CSEL" -"MUL"
|
|
// ppc64x:"CLRLSLDI [$]56, R[0-9]+, [$]1" "OR" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
func cmovmathor2else(a uint, b bool) uint {
|
|
if !b {
|
|
a |= 2
|
|
}
|
|
// amd64:"ORQ" -"CMOV"
|
|
// arm64:"ORR R[0-9]+<<1" -"CSEL" -"MUL"
|
|
// ppc64x:"CLRLSLDI [$]56, R[0-9]+, [$]1" "OR" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
|
|
func cmovmathor9223372036854775808(a uint, b bool) uint {
|
|
if b {
|
|
a |= 1 << 63
|
|
}
|
|
// amd64:"ORQ" -"CMOV"
|
|
// arm64:"ORR R[0-9]+<<63" -"CSEL" -"MUL"
|
|
// ppc64x:"SLD [$]63" "OR" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
func cmovmathor9223372036854775808else(a uint, b bool) uint {
|
|
if !b {
|
|
a |= 1 << 63
|
|
}
|
|
// amd64:"ORQ" -"CMOV"
|
|
// arm64:"ORR R[0-9]+<<63" -"CSEL" -"MUL"
|
|
// ppc64x:"SLD [$]63" "OR" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
|
|
func cmovmathxor(a uint, b bool) uint {
|
|
if b {
|
|
a ^= 1
|
|
}
|
|
// amd64:"XORQ" -"CMOV"
|
|
// arm64:"EOR" -"CSEL"
|
|
// ppc64x:"XOR" -"ISEL"
|
|
// wasm:"I64Xor" -"Select"
|
|
return a
|
|
}
|
|
func cmovmathxorelse(a uint, b bool) uint {
|
|
if !b {
|
|
a ^= 1
|
|
}
|
|
// amd64:"XORQ" -"CMOV"
|
|
// arm64:"EOR" -"CSEL"
|
|
// ppc64x:"XOR" -"ISEL"
|
|
// wasm:"I64Xor" -"Select"
|
|
return a
|
|
}
|
|
|
|
func cmovmathxor2(a uint, b bool) uint {
|
|
if b {
|
|
a ^= 2
|
|
}
|
|
// amd64:"XORQ" -"CMOV"
|
|
// arm64:"EOR R[0-9]+<<1" -"CSEL" -"MUL"
|
|
// ppc64x:"CLRLSLDI [$]56, R[0-9]+, [$]1" "XOR" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
func cmovmathxor2else(a uint, b bool) uint {
|
|
if !b {
|
|
a ^= 2
|
|
}
|
|
// amd64:"XORQ" -"CMOV"
|
|
// arm64:"EOR R[0-9]+<<1" -"CSEL" -"MUL"
|
|
// ppc64x:"CLRLSLDI [$]56, R[0-9]+, [$]1" "XOR" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
|
|
func cmovmathxor9223372036854775808(a uint, b bool) uint {
|
|
if b {
|
|
a ^= 1 << 63
|
|
}
|
|
// amd64:"XORQ" -"CMOV"
|
|
// arm64:"EOR R[0-9]+<<63" -"CSEL" -"MUL"
|
|
// ppc64x:"SLD [$]63" "XOR" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
func cmovmathxor9223372036854775808else(a uint, b bool) uint {
|
|
if !b {
|
|
a ^= 1 << 63
|
|
}
|
|
// amd64:"XORQ" -"CMOV"
|
|
// arm64:"EOR R[0-9]+<<63" -"CSEL" -"MUL"
|
|
// ppc64x:"SLD [$]63" "XOR" -"ISEL" -"MUL"
|
|
return a
|
|
}
|
|
|
|
func branchlessBoolToUint8(b bool) (r uint8) {
|
|
if b {
|
|
r = 1
|
|
}
|
|
return
|
|
}
|
|
|
|
func cmovFromMulFromFlags64(x uint64, b bool) uint64 {
|
|
// amd64:-"MOVB.ZX"
|
|
r := uint64(branchlessBoolToUint8(b))
|
|
// amd64:"CMOV" -"MOVB.ZX" -"MUL"
|
|
return x * r
|
|
}
|
|
func cmovFromMulFromFlags64sext(x int64, b bool) int64 {
|
|
// amd64:-"MOVB.ZX"
|
|
r := int64(int8(branchlessBoolToUint8(b)))
|
|
// amd64:"CMOV" -"MOVB.ZX" -"MUL"
|
|
return x * r
|
|
}
|
|
|
|
func constantTimeSelect(v, x, y int) int {
|
|
// amd64:"CMOVQ"
|
|
// arm64:"CSEL"
|
|
// riscv64/rva20u64,riscv64/rva22u64:"SNEZ" "NEG" "AND" "OR"
|
|
// riscv64/rva23u64:"CZERONEZ" "CZEROEQZ" "OR" -"SNEZ" -"NEG" -"AND"
|
|
return subtle.ConstantTimeSelect(v, x, y)
|
|
}
|
|
|
|
func issue76056fieldReduceOnceSub32(a uint32) uint32 {
|
|
const q = 8380417 // 2²³ - 2¹³ + 1
|
|
// FIXME: the compiler struggles with Sub32 since it's not intriscified.
|
|
x, b := bits.Sub32(a, q, 0)
|
|
// FIXME: prove doesn't rewrite this multiply to a condselect because it doesn't know that b is always 0 or 1.
|
|
return x + b*q
|
|
}
|
|
|
|
func issue76056fieldReduceOnce2Sub32(a uint32) uint32 {
|
|
const q = 8380417 // 2²³ - 2¹³ + 1
|
|
// FIXME: the compiler struggles with Sub32 since it's not intriscified.
|
|
x, b := bits.Sub32(a, q, 0)
|
|
// riscv64/rva20u64,riscv64/rva22u64:"SNEZ" "NEG" "AND" "OR"
|
|
// riscv64/rva23u64:"CZERONEZ" "CZEROEQZ" "OR" -"SNEZ" -"NEG" -"AND"
|
|
return uint32(subtle.ConstantTimeSelect(int(b), int(a), int(x)))
|
|
}
|
|
|
|
func issue76056fieldReduceOnceSub64(a uint32) uint32 {
|
|
const q = 8380417 // 2²³ - 2¹³ + 1
|
|
x, b := bits.Sub64(uint64(a), q, 0)
|
|
// FIXME: prove doesn't rewrite this multiply to a condselect because it doesn't know that b is always 0 or 1.
|
|
return uint32(x) + uint32(b)*q
|
|
}
|
|
|
|
func issue76056fieldReduceOnce2Sub64(a uint32) uint32 {
|
|
const q = 8380417 // 2²³ - 2¹³ + 1
|
|
// amd64:"SUB" -"TEST" -"SBB"
|
|
x, b := bits.Sub64(uint64(a), q, 0)
|
|
// amd64:"CMOV" -"TEST" -"SBB"
|
|
// riscv64/rva20u64,riscv64/rva22u64:"SNEZ" "NEG" "AND" "OR"
|
|
// riscv64/rva23u64:"CZERONEZ" "CZEROEQZ" "OR" -"SNEZ" -"NEG" -"AND"
|
|
return uint32(subtle.ConstantTimeSelect(int(b), int(a), int(x)))
|
|
}
|