cmd/compile: recognize bit test patterns on amd64

Updates #18943

Change-Id: If3080d6133bb6d2710b57294da24c90251ab4e08
Reviewed-on: https://go-review.googlesource.com/36329
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Keith Randall <khr@golang.org>

diff --git a/src/cmd/compile/internal/amd64/prog.go b/src/cmd/compile/internal/amd64/prog.go
index 6dff09329b5f197c8b87ab52c86b2217160ecc58..1a71cc56d7242433a2747647305cf8f08eefbd40 100644 (file)
--- a/src/cmd/compile/internal/amd64/prog.go
+++ b/src/cmd/compile/internal/amd64/prog.go
@@ -72,6 +72,8 @@ var progtable = [x86.ALAST & obj.AMask]gc.ProgInfo{
        x86.ACMOVWEQ & obj.AMask: {Flags: gc.SizeW | gc.LeftRead | RightRdwr | gc.UseCarry},
        x86.ACMOVWNE & obj.AMask: {Flags: gc.SizeW | gc.LeftRead | RightRdwr | gc.UseCarry},
 
+       x86.ABTL & obj.AMask:       {Flags: gc.SizeL | gc.LeftRead | gc.RightRead | gc.SetCarry},
+       x86.ABTQ & obj.AMask:       {Flags: gc.SizeQ | gc.LeftRead | gc.RightRead | gc.SetCarry},
        x86.ACMPB & obj.AMask:      {Flags: gc.SizeB | gc.LeftRead | gc.RightRead | gc.SetCarry},
        x86.ACMPL & obj.AMask:      {Flags: gc.SizeL | gc.LeftRead | gc.RightRead | gc.SetCarry},
        x86.ACMPQ & obj.AMask:      {Flags: gc.SizeQ | gc.LeftRead | gc.RightRead | gc.SetCarry},

diff --git a/src/cmd/compile/internal/amd64/ssa.go b/src/cmd/compile/internal/amd64/ssa.go
index d930d433d1460b290c712713403772d2c616282e..b403e6d0c99cb2b8ba1cdbf0938be770ce83a02d 100644 (file)
--- a/src/cmd/compile/internal/amd64/ssa.go
+++ b/src/cmd/compile/internal/amd64/ssa.go
@@ -471,7 +471,8 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
                p.To.Type = obj.TYPE_REG
                p.To.Reg = v.Reg()
        case ssa.OpAMD64CMPQ, ssa.OpAMD64CMPL, ssa.OpAMD64CMPW, ssa.OpAMD64CMPB,
-               ssa.OpAMD64TESTQ, ssa.OpAMD64TESTL, ssa.OpAMD64TESTW, ssa.OpAMD64TESTB:
+               ssa.OpAMD64TESTQ, ssa.OpAMD64TESTL, ssa.OpAMD64TESTW, ssa.OpAMD64TESTB,
+               ssa.OpAMD64BTL, ssa.OpAMD64BTQ:
                opregreg(v.Op.Asm(), v.Args[1].Reg(), v.Args[0].Reg())
        case ssa.OpAMD64UCOMISS, ssa.OpAMD64UCOMISD:
                // Go assembler has swapped operands for UCOMISx relative to CMP,
@@ -483,7 +484,8 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
                p.From.Reg = v.Args[0].Reg()
                p.To.Type = obj.TYPE_CONST
                p.To.Offset = v.AuxInt
-       case ssa.OpAMD64TESTQconst, ssa.OpAMD64TESTLconst, ssa.OpAMD64TESTWconst, ssa.OpAMD64TESTBconst:
+       case ssa.OpAMD64TESTQconst, ssa.OpAMD64TESTLconst, ssa.OpAMD64TESTWconst, ssa.OpAMD64TESTBconst,
+               ssa.OpAMD64BTLconst, ssa.OpAMD64BTQconst:
                p := gc.Prog(v.Op.Asm())
                p.From.Type = obj.TYPE_CONST
                p.From.Offset = v.AuxInt

diff --git a/src/cmd/compile/internal/gc/asm_test.go b/src/cmd/compile/internal/gc/asm_test.go
index 4525ba3f0cebc9f4d438daa5257d1c31ad82f70f..33e9f47b089f676b5097fa5ef7944873c8328938 100644 (file)
--- a/src/cmd/compile/internal/gc/asm_test.go
+++ b/src/cmd/compile/internal/gc/asm_test.go
@@ -504,6 +504,45 @@ var linuxAMD64Tests = []*asmTest{
                `,
                []string{"\"abc\""},
        },
+       // Bit test ops on amd64, issue 18943.
+       {
+               `
+               func f37(a, b uint64) int {
+                       if a&(1<<(b&63)) != 0 {
+                               return 1
+                       }
+                       return -1
+               }
+               `,
+               []string{"\tBTQ\t"},
+       },
+       {
+               `
+               func f38(a, b uint64) bool {
+                       return a&(1<<(b&63)) != 0
+               }
+               `,
+               []string{"\tBTQ\t"},
+       },
+       {
+               `
+               func f39(a uint64) int {
+                       if a&(1<<60) != 0 {
+                               return 1
+                       }
+                       return -1
+               }
+               `,
+               []string{"\tBTQ\t\\$60"},
+       },
+       {
+               `
+               func f40(a uint64) bool {
+                       return a&(1<<60) != 0
+               }
+               `,
+               []string{"\tBTQ\t\\$60"},
+       },
 }
 
 var linux386Tests = []*asmTest{

diff --git a/src/cmd/compile/internal/ssa/gen/AMD64.rules b/src/cmd/compile/internal/ssa/gen/AMD64.rules
index 24e6494dcf6993df75ea0d95147c99bcc498bfce..fa1359fecf0ba90262af030adc85e253389bbc56 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/AMD64.rules
+++ b/src/cmd/compile/internal/ssa/gen/AMD64.rules
@@ -518,6 +518,37 @@
 (NE (TESTB (SETA  cmp) (SETA  cmp)) yes no) -> (UGT cmp yes no)
 (NE (TESTB (SETAE cmp) (SETAE cmp)) yes no) -> (UGE cmp yes no)
 
+// Normalize TESTx argument order for BTx rewrites below.
+(TESTQ y x:(SHLQ _ _)) && y.Op != OpAMD64SHLQ -> (TESTQ x y)
+(TESTL y x:(SHLL _ _)) && y.Op != OpAMD64SHLL -> (TESTL x y)
+
+// Recognize bit tests: a&(1<<b) != 0 for b suitably bounded
+// Note that ULT and SETB check the carry flag; they are identical to CS and SETCS.
+// Same, mutatis mutandis, for UGE and SETAE, and CC and SETCC.
+(NE (TESTL (SHLL (MOVLconst [1]) x) y)) && !config.nacl -> (ULT (BTL x y))
+(EQ (TESTL (SHLL (MOVLconst [1]) x) y)) && !config.nacl -> (UGE (BTL x y))
+(NE (TESTQ (SHLQ (MOVQconst [1]) x) y)) && !config.nacl -> (ULT (BTQ x y))
+(EQ (TESTQ (SHLQ (MOVQconst [1]) x) y)) && !config.nacl -> (UGE (BTQ x y))
+(NE (TESTLconst [c] x)) && isPowerOfTwo(c) && log2(c) < 32 && !config.nacl -> (ULT (BTLconst [log2(c)] x))
+(EQ (TESTLconst [c] x)) && isPowerOfTwo(c) && log2(c) < 32 && !config.nacl -> (UGE (BTLconst [log2(c)] x))
+(NE (TESTQconst [c] x)) && isPowerOfTwo(c) && log2(c) < 64 && !config.nacl -> (ULT (BTQconst [log2(c)] x))
+(EQ (TESTQconst [c] x)) && isPowerOfTwo(c) && log2(c) < 64 && !config.nacl -> (UGE (BTQconst [log2(c)] x))
+(NE (TESTQ (MOVQconst [c]) x)) && isPowerOfTwo(c) && log2(c) < 64 && !config.nacl -> (ULT (BTQconst [log2(c)] x))
+(EQ (TESTQ (MOVQconst [c]) x)) && isPowerOfTwo(c) && log2(c) < 64 && !config.nacl -> (UGE (BTQconst [log2(c)] x))
+(SETNE (TESTL (SHLL (MOVLconst [1]) x) y)) && !config.nacl -> (SETB  (BTL x y))
+(SETEQ (TESTL (SHLL (MOVLconst [1]) x) y)) && !config.nacl -> (SETAE (BTL x y))
+(SETNE (TESTQ (SHLQ (MOVQconst [1]) x) y)) && !config.nacl -> (SETB  (BTQ x y))
+(SETEQ (TESTQ (SHLQ (MOVQconst [1]) x) y)) && !config.nacl -> (SETAE (BTQ x y))
+(SETNE (TESTLconst [c] x)) && isPowerOfTwo(c) && log2(c) < 32 && !config.nacl -> (SETB  (BTLconst [log2(c)] x))
+(SETEQ (TESTLconst [c] x)) && isPowerOfTwo(c) && log2(c) < 32 && !config.nacl -> (SETAE (BTLconst [log2(c)] x))
+(SETNE (TESTQconst [c] x)) && isPowerOfTwo(c) && log2(c) < 64 && !config.nacl -> (SETB  (BTQconst [log2(c)] x))
+(SETEQ (TESTQconst [c] x)) && isPowerOfTwo(c) && log2(c) < 64 && !config.nacl -> (SETAE (BTQconst [log2(c)] x))
+(SETNE (TESTQ (MOVQconst [c]) x)) && isPowerOfTwo(c) && log2(c) < 64 && !config.nacl -> (SETB  (BTQconst [log2(c)] x))
+(SETEQ (TESTQ (MOVQconst [c]) x)) && isPowerOfTwo(c) && log2(c) < 64 && !config.nacl -> (SETAE (BTQconst [log2(c)] x))
+
+// Convert BTQconst to BTLconst if possible. It has a shorter encoding.
+(BTQconst [c] x) && c < 32 -> (BTLconst [c] x)
+
 // Special case for floating point - LF/LEF not generated
 (NE (TESTB (SETGF  cmp) (SETGF  cmp)) yes no) -> (UGT  cmp yes no)
 (NE (TESTB (SETGEF cmp) (SETGEF cmp)) yes no) -> (UGE  cmp yes no)
@@ -1380,6 +1411,16 @@
 (CMPWconst (ANDLconst [c] x) [0]) -> (TESTWconst [int64(int16(c))] x)
 (CMPBconst (ANDLconst [c] x) [0]) -> (TESTBconst [int64(int8(c))] x)
 
+// Convert TESTx to TESTxconst if possible.
+(TESTQ (MOVQconst [c]) x) && c < 1<<31 -> (TESTQconst [c] x)
+(TESTL (MOVLconst [c]) x) -> (TESTLconst [c] x)
+(TESTW (MOVLconst [c]) x) -> (TESTWconst [c] x)
+(TESTB (MOVLconst [c]) x) -> (TESTBconst [c] x)
+(TESTQ x (MOVQconst [c])) && c < 1<<31 -> (TESTQconst [c] x)
+(TESTL x (MOVLconst [c])) -> (TESTLconst [c] x)
+(TESTW x (MOVLconst [c])) -> (TESTWconst [c] x)
+(TESTB x (MOVLconst [c])) -> (TESTBconst [c] x)
+
 // TEST %reg,%reg is shorter than CMP
 (CMPQconst x [0]) -> (TESTQ x x)
 (CMPLconst x [0]) -> (TESTL x x)

diff --git a/src/cmd/compile/internal/ssa/gen/AMD64Ops.go b/src/cmd/compile/internal/ssa/gen/AMD64Ops.go
index 5e8fd4bb89bde7451a1cb28a9b2282950f988221..0af9d004eca7490a1af190a1f2cb60243fadee7c 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/AMD64Ops.go
+++ b/src/cmd/compile/internal/ssa/gen/AMD64Ops.go
@@ -250,6 +250,11 @@ func init() {
                {name: "UCOMISS", argLength: 2, reg: fp2flags, asm: "UCOMISS", typ: "Flags"}, // arg0 compare to arg1, f32
                {name: "UCOMISD", argLength: 2, reg: fp2flags, asm: "UCOMISD", typ: "Flags"}, // arg0 compare to arg1, f64
 
+               {name: "BTL", argLength: 2, reg: gp2flags, asm: "BTL", typ: "Flags"},                   // test whether bit arg0 % 32 in arg1 is set
+               {name: "BTQ", argLength: 2, reg: gp2flags, asm: "BTQ", typ: "Flags"},                   // test whether bit arg0 % 64 in arg1 is set
+               {name: "BTLconst", argLength: 1, reg: gp1flags, asm: "BTL", typ: "Flags", aux: "Int8"}, // test whether bit auxint in arg0 is set, 0 <= auxint < 32
+               {name: "BTQconst", argLength: 1, reg: gp1flags, asm: "BTQ", typ: "Flags", aux: "Int8"}, // test whether bit auxint in arg0 is set, 0 <= auxint < 64
+
                {name: "TESTQ", argLength: 2, reg: gp2flags, asm: "TESTQ", typ: "Flags"},                    // (arg0 & arg1) compare to 0
                {name: "TESTL", argLength: 2, reg: gp2flags, asm: "TESTL", typ: "Flags"},                    // (arg0 & arg1) compare to 0
                {name: "TESTW", argLength: 2, reg: gp2flags, asm: "TESTW", typ: "Flags"},                    // (arg0 & arg1) compare to 0

diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go
index 9e103761284d2c5f2f8c2600b1b17aa810c431ea..d8ede8d6336d6d7ab5d870d368ac45e04cbb1068 100644 (file)
--- a/src/cmd/compile/internal/ssa/opGen.go
+++ b/src/cmd/compile/internal/ssa/opGen.go
@@ -491,6 +491,10 @@ const (
        OpAMD64CMPBconst
        OpAMD64UCOMISS
        OpAMD64UCOMISD
+       OpAMD64BTL
+       OpAMD64BTQ
+       OpAMD64BTLconst
+       OpAMD64BTQconst
        OpAMD64TESTQ
        OpAMD64TESTL
        OpAMD64TESTW
@@ -5550,6 +5554,50 @@ var opcodeTable = [...]opInfo{
                        },
                },
        },
+       {
+               name:   "BTL",
+               argLen: 2,
+               asm:    x86.ABTL,
+               reg: regInfo{
+                       inputs: []inputInfo{
+                               {0, 65535}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+                               {1, 65535}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+                       },
+               },
+       },
+       {
+               name:   "BTQ",
+               argLen: 2,
+               asm:    x86.ABTQ,
+               reg: regInfo{
+                       inputs: []inputInfo{
+                               {0, 65535}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+                               {1, 65535}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+                       },
+               },
+       },
+       {
+               name:    "BTLconst",
+               auxType: auxInt8,
+               argLen:  1,
+               asm:     x86.ABTL,
+               reg: regInfo{
+                       inputs: []inputInfo{
+                               {0, 65535}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+                       },
+               },
+       },
+       {
+               name:    "BTQconst",
+               auxType: auxInt8,
+               argLen:  1,
+               asm:     x86.ABTQ,
+               reg: regInfo{
+                       inputs: []inputInfo{
+                               {0, 65535}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R14 R15
+                       },
+               },
+       },
        {
                name:   "TESTQ",
                argLen: 2,

diff --git a/src/cmd/compile/internal/ssa/rewriteAMD64.go b/src/cmd/compile/internal/ssa/rewriteAMD64.go
index f0fc186391065c92643ceeca209f6b7dc54d1483..49cf0d166536c6fc9411d5777a6f73d26b89f597 100644 (file)
--- a/src/cmd/compile/internal/ssa/rewriteAMD64.go
+++ b/src/cmd/compile/internal/ssa/rewriteAMD64.go
@@ -28,6 +28,8 @@ func rewriteValueAMD64(v *Value, config *Config) bool {
                return rewriteValueAMD64_OpAMD64ANDQ(v, config)
        case OpAMD64ANDQconst:
                return rewriteValueAMD64_OpAMD64ANDQconst(v, config)
+       case OpAMD64BTQconst:
+               return rewriteValueAMD64_OpAMD64BTQconst(v, config)
        case OpAMD64CMPB:
                return rewriteValueAMD64_OpAMD64CMPB(v, config)
        case OpAMD64CMPBconst:
@@ -288,6 +290,14 @@ func rewriteValueAMD64(v *Value, config *Config) bool {
                return rewriteValueAMD64_OpAMD64SUBSD(v, config)
        case OpAMD64SUBSS:
                return rewriteValueAMD64_OpAMD64SUBSS(v, config)
+       case OpAMD64TESTB:
+               return rewriteValueAMD64_OpAMD64TESTB(v, config)
+       case OpAMD64TESTL:
+               return rewriteValueAMD64_OpAMD64TESTL(v, config)
+       case OpAMD64TESTQ:
+               return rewriteValueAMD64_OpAMD64TESTQ(v, config)
+       case OpAMD64TESTW:
+               return rewriteValueAMD64_OpAMD64TESTW(v, config)
        case OpAMD64XADDLlock:
                return rewriteValueAMD64_OpAMD64XADDLlock(v, config)
        case OpAMD64XADDQlock:
@@ -2160,6 +2170,25 @@ func rewriteValueAMD64_OpAMD64ANDQconst(v *Value, config *Config) bool {
        }
        return false
 }
+func rewriteValueAMD64_OpAMD64BTQconst(v *Value, config *Config) bool {
+       b := v.Block
+       _ = b
+       // match: (BTQconst [c] x)
+       // cond: c < 32
+       // result: (BTLconst [c] x)
+       for {
+               c := v.AuxInt
+               x := v.Args[0]
+               if !(c < 32) {
+                       break
+               }
+               v.reset(OpAMD64BTLconst)
+               v.AuxInt = c
+               v.AddArg(x)
+               return true
+       }
+       return false
+}
 func rewriteValueAMD64_OpAMD64CMPB(v *Value, config *Config) bool {
        b := v.Block
        _ = b
@@ -14748,6 +14777,132 @@ func rewriteValueAMD64_OpAMD64SETBE(v *Value, config *Config) bool {
 func rewriteValueAMD64_OpAMD64SETEQ(v *Value, config *Config) bool {
        b := v.Block
        _ = b
+       // match: (SETEQ (TESTL (SHLL (MOVLconst [1]) x) y))
+       // cond: !config.nacl
+       // result: (SETAE (BTL x y))
+       for {
+               v_0 := v.Args[0]
+               if v_0.Op != OpAMD64TESTL {
+                       break
+               }
+               v_0_0 := v_0.Args[0]
+               if v_0_0.Op != OpAMD64SHLL {
+                       break
+               }
+               v_0_0_0 := v_0_0.Args[0]
+               if v_0_0_0.Op != OpAMD64MOVLconst {
+                       break
+               }
+               if v_0_0_0.AuxInt != 1 {
+                       break
+               }
+               x := v_0_0.Args[1]
+               y := v_0.Args[1]
+               if !(!config.nacl) {
+                       break
+               }
+               v.reset(OpAMD64SETAE)
+               v0 := b.NewValue0(v.Pos, OpAMD64BTL, TypeFlags)
+               v0.AddArg(x)
+               v0.AddArg(y)
+               v.AddArg(v0)
+               return true
+       }
+       // match: (SETEQ (TESTQ (SHLQ (MOVQconst [1]) x) y))
+       // cond: !config.nacl
+       // result: (SETAE (BTQ x y))
+       for {
+               v_0 := v.Args[0]
+               if v_0.Op != OpAMD64TESTQ {
+                       break
+               }
+               v_0_0 := v_0.Args[0]
+               if v_0_0.Op != OpAMD64SHLQ {
+                       break
+               }
+               v_0_0_0 := v_0_0.Args[0]
+               if v_0_0_0.Op != OpAMD64MOVQconst {
+                       break
+               }
+               if v_0_0_0.AuxInt != 1 {
+                       break
+               }
+               x := v_0_0.Args[1]
+               y := v_0.Args[1]
+               if !(!config.nacl) {
+                       break
+               }
+               v.reset(OpAMD64SETAE)
+               v0 := b.NewValue0(v.Pos, OpAMD64BTQ, TypeFlags)
+               v0.AddArg(x)
+               v0.AddArg(y)
+               v.AddArg(v0)
+               return true
+       }
+       // match: (SETEQ (TESTLconst [c] x))
+       // cond: isPowerOfTwo(c) && log2(c) < 32 && !config.nacl
+       // result: (SETAE (BTLconst [log2(c)] x))
+       for {
+               v_0 := v.Args[0]
+               if v_0.Op != OpAMD64TESTLconst {
+                       break
+               }
+               c := v_0.AuxInt
+               x := v_0.Args[0]
+               if !(isPowerOfTwo(c) && log2(c) < 32 && !config.nacl) {
+                       break
+               }
+               v.reset(OpAMD64SETAE)
+               v0 := b.NewValue0(v.Pos, OpAMD64BTLconst, TypeFlags)
+               v0.AuxInt = log2(c)
+               v0.AddArg(x)
+               v.AddArg(v0)
+               return true
+       }
+       // match: (SETEQ (TESTQconst [c] x))
+       // cond: isPowerOfTwo(c) && log2(c) < 64 && !config.nacl
+       // result: (SETAE (BTQconst [log2(c)] x))
+       for {
+               v_0 := v.Args[0]
+               if v_0.Op != OpAMD64TESTQconst {
+                       break
+               }
+               c := v_0.AuxInt
+               x := v_0.Args[0]
+               if !(isPowerOfTwo(c) && log2(c) < 64 && !config.nacl) {
+                       break
+               }
+               v.reset(OpAMD64SETAE)
+               v0 := b.NewValue0(v.Pos, OpAMD64BTQconst, TypeFlags)
+               v0.AuxInt = log2(c)
+               v0.AddArg(x)
+               v.AddArg(v0)
+               return true
+       }
+       // match: (SETEQ (TESTQ (MOVQconst [c]) x))
+       // cond: isPowerOfTwo(c) && log2(c) < 64 && !config.nacl
+       // result: (SETAE (BTQconst [log2(c)] x))
+       for {
+               v_0 := v.Args[0]
+               if v_0.Op != OpAMD64TESTQ {
+                       break
+               }
+               v_0_0 := v_0.Args[0]
+               if v_0_0.Op != OpAMD64MOVQconst {
+                       break
+               }
+               c := v_0_0.AuxInt
+               x := v_0.Args[1]
+               if !(isPowerOfTwo(c) && log2(c) < 64 && !config.nacl) {
+                       break
+               }
+               v.reset(OpAMD64SETAE)
+               v0 := b.NewValue0(v.Pos, OpAMD64BTQconst, TypeFlags)
+               v0.AuxInt = log2(c)
+               v0.AddArg(x)
+               v.AddArg(v0)
+               return true
+       }
        // match: (SETEQ (InvertFlags x))
        // cond:
        // result: (SETEQ x)
@@ -15138,6 +15293,132 @@ func rewriteValueAMD64_OpAMD64SETLE(v *Value, config *Config) bool {
 func rewriteValueAMD64_OpAMD64SETNE(v *Value, config *Config) bool {
        b := v.Block
        _ = b
+       // match: (SETNE (TESTL (SHLL (MOVLconst [1]) x) y))
+       // cond: !config.nacl
+       // result: (SETB  (BTL x y))
+       for {
+               v_0 := v.Args[0]
+               if v_0.Op != OpAMD64TESTL {
+                       break
+               }
+               v_0_0 := v_0.Args[0]
+               if v_0_0.Op != OpAMD64SHLL {
+                       break
+               }
+               v_0_0_0 := v_0_0.Args[0]
+               if v_0_0_0.Op != OpAMD64MOVLconst {
+                       break
+               }
+               if v_0_0_0.AuxInt != 1 {
+                       break
+               }
+               x := v_0_0.Args[1]
+               y := v_0.Args[1]
+               if !(!config.nacl) {
+                       break
+               }
+               v.reset(OpAMD64SETB)
+               v0 := b.NewValue0(v.Pos, OpAMD64BTL, TypeFlags)
+               v0.AddArg(x)
+               v0.AddArg(y)
+               v.AddArg(v0)
+               return true
+       }
+       // match: (SETNE (TESTQ (SHLQ (MOVQconst [1]) x) y))
+       // cond: !config.nacl
+       // result: (SETB  (BTQ x y))
+       for {
+               v_0 := v.Args[0]
+               if v_0.Op != OpAMD64TESTQ {
+                       break
+               }
+               v_0_0 := v_0.Args[0]
+               if v_0_0.Op != OpAMD64SHLQ {
+                       break
+               }
+               v_0_0_0 := v_0_0.Args[0]
+               if v_0_0_0.Op != OpAMD64MOVQconst {
+                       break
+               }
+               if v_0_0_0.AuxInt != 1 {
+                       break
+               }
+               x := v_0_0.Args[1]
+               y := v_0.Args[1]
+               if !(!config.nacl) {
+                       break
+               }
+               v.reset(OpAMD64SETB)
+               v0 := b.NewValue0(v.Pos, OpAMD64BTQ, TypeFlags)
+               v0.AddArg(x)
+               v0.AddArg(y)
+               v.AddArg(v0)
+               return true
+       }
+       // match: (SETNE (TESTLconst [c] x))
+       // cond: isPowerOfTwo(c) && log2(c) < 32 && !config.nacl
+       // result: (SETB  (BTLconst [log2(c)] x))
+       for {
+               v_0 := v.Args[0]
+               if v_0.Op != OpAMD64TESTLconst {
+                       break
+               }
+               c := v_0.AuxInt
+               x := v_0.Args[0]
+               if !(isPowerOfTwo(c) && log2(c) < 32 && !config.nacl) {
+                       break
+               }
+               v.reset(OpAMD64SETB)
+               v0 := b.NewValue0(v.Pos, OpAMD64BTLconst, TypeFlags)
+               v0.AuxInt = log2(c)
+               v0.AddArg(x)
+               v.AddArg(v0)
+               return true
+       }
+       // match: (SETNE (TESTQconst [c] x))
+       // cond: isPowerOfTwo(c) && log2(c) < 64 && !config.nacl
+       // result: (SETB  (BTQconst [log2(c)] x))
+       for {
+               v_0 := v.Args[0]
+               if v_0.Op != OpAMD64TESTQconst {
+                       break
+               }
+               c := v_0.AuxInt
+               x := v_0.Args[0]
+               if !(isPowerOfTwo(c) && log2(c) < 64 && !config.nacl) {
+                       break
+               }
+               v.reset(OpAMD64SETB)
+               v0 := b.NewValue0(v.Pos, OpAMD64BTQconst, TypeFlags)
+               v0.AuxInt = log2(c)
+               v0.AddArg(x)
+               v.AddArg(v0)
+               return true
+       }
+       // match: (SETNE (TESTQ (MOVQconst [c]) x))
+       // cond: isPowerOfTwo(c) && log2(c) < 64 && !config.nacl
+       // result: (SETB  (BTQconst [log2(c)] x))
+       for {
+               v_0 := v.Args[0]
+               if v_0.Op != OpAMD64TESTQ {
+                       break
+               }
+               v_0_0 := v_0.Args[0]
+               if v_0_0.Op != OpAMD64MOVQconst {
+                       break
+               }
+               c := v_0_0.AuxInt
+               x := v_0.Args[1]
+               if !(isPowerOfTwo(c) && log2(c) < 64 && !config.nacl) {
+                       break
+               }
+               v.reset(OpAMD64SETB)
+               v0 := b.NewValue0(v.Pos, OpAMD64BTQconst, TypeFlags)
+               v0.AuxInt = log2(c)
+               v0.AddArg(x)
+               v.AddArg(v0)
+               return true
+       }
        // match: (SETNE (InvertFlags x))
        // cond:
        // result: (SETNE x)
@@ -15983,6 +16264,186 @@ func rewriteValueAMD64_OpAMD64SUBSS(v *Value, config *Config) bool {
        }
        return false
 }
+func rewriteValueAMD64_OpAMD64TESTB(v *Value, config *Config) bool {
+       b := v.Block
+       _ = b
+       // match: (TESTB (MOVLconst [c]) x)
+       // cond:
+       // result: (TESTBconst [c] x)
+       for {
+               v_0 := v.Args[0]
+               if v_0.Op != OpAMD64MOVLconst {
+                       break
+               }
+               c := v_0.AuxInt
+               x := v.Args[1]
+               v.reset(OpAMD64TESTBconst)
+               v.AuxInt = c
+               v.AddArg(x)
+               return true
+       }
+       // match: (TESTB x (MOVLconst [c]))
+       // cond:
+       // result: (TESTBconst [c] x)
+       for {
+               x := v.Args[0]
+               v_1 := v.Args[1]
+               if v_1.Op != OpAMD64MOVLconst {
+                       break
+               }
+               c := v_1.AuxInt
+               v.reset(OpAMD64TESTBconst)
+               v.AuxInt = c
+               v.AddArg(x)
+               return true
+       }
+       return false
+}
+func rewriteValueAMD64_OpAMD64TESTL(v *Value, config *Config) bool {
+       b := v.Block
+       _ = b
+       // match: (TESTL y x:(SHLL _ _))
+       // cond: y.Op != OpAMD64SHLL
+       // result: (TESTL x y)
+       for {
+               y := v.Args[0]
+               x := v.Args[1]
+               if x.Op != OpAMD64SHLL {
+                       break
+               }
+               if !(y.Op != OpAMD64SHLL) {
+                       break
+               }
+               v.reset(OpAMD64TESTL)
+               v.AddArg(x)
+               v.AddArg(y)
+               return true
+       }
+       // match: (TESTL (MOVLconst [c]) x)
+       // cond:
+       // result: (TESTLconst [c] x)
+       for {
+               v_0 := v.Args[0]
+               if v_0.Op != OpAMD64MOVLconst {
+                       break
+               }
+               c := v_0.AuxInt
+               x := v.Args[1]
+               v.reset(OpAMD64TESTLconst)
+               v.AuxInt = c
+               v.AddArg(x)
+               return true
+       }
+       // match: (TESTL x (MOVLconst [c]))
+       // cond:
+       // result: (TESTLconst [c] x)
+       for {
+               x := v.Args[0]
+               v_1 := v.Args[1]
+               if v_1.Op != OpAMD64MOVLconst {
+                       break
+               }
+               c := v_1.AuxInt
+               v.reset(OpAMD64TESTLconst)
+               v.AuxInt = c
+               v.AddArg(x)
+               return true
+       }
+       return false
+}
+func rewriteValueAMD64_OpAMD64TESTQ(v *Value, config *Config) bool {
+       b := v.Block
+       _ = b
+       // match: (TESTQ y x:(SHLQ _ _))
+       // cond: y.Op != OpAMD64SHLQ
+       // result: (TESTQ x y)
+       for {
+               y := v.Args[0]
+               x := v.Args[1]
+               if x.Op != OpAMD64SHLQ {
+                       break
+               }
+               if !(y.Op != OpAMD64SHLQ) {
+                       break
+               }
+               v.reset(OpAMD64TESTQ)
+               v.AddArg(x)
+               v.AddArg(y)
+               return true
+       }
+       // match: (TESTQ (MOVQconst [c]) x)
+       // cond: c < 1<<31
+       // result: (TESTQconst [c] x)
+       for {
+               v_0 := v.Args[0]
+               if v_0.Op != OpAMD64MOVQconst {
+                       break
+               }
+               c := v_0.AuxInt
+               x := v.Args[1]
+               if !(c < 1<<31) {
+                       break
+               }
+               v.reset(OpAMD64TESTQconst)
+               v.AuxInt = c
+               v.AddArg(x)
+               return true
+       }
+       // match: (TESTQ x (MOVQconst [c]))
+       // cond: c < 1<<31
+       // result: (TESTQconst [c] x)
+       for {
+               x := v.Args[0]
+               v_1 := v.Args[1]
+               if v_1.Op != OpAMD64MOVQconst {
+                       break
+               }
+               c := v_1.AuxInt
+               if !(c < 1<<31) {
+                       break
+               }
+               v.reset(OpAMD64TESTQconst)
+               v.AuxInt = c
+               v.AddArg(x)
+               return true
+       }
+       return false
+}
+func rewriteValueAMD64_OpAMD64TESTW(v *Value, config *Config) bool {
+       b := v.Block
+       _ = b
+       // match: (TESTW (MOVLconst [c]) x)
+       // cond:
+       // result: (TESTWconst [c] x)
+       for {
+               v_0 := v.Args[0]
+               if v_0.Op != OpAMD64MOVLconst {
+                       break
+               }
+               c := v_0.AuxInt
+               x := v.Args[1]
+               v.reset(OpAMD64TESTWconst)
+               v.AuxInt = c
+               v.AddArg(x)
+               return true
+       }
+       // match: (TESTW x (MOVLconst [c]))
+       // cond:
+       // result: (TESTWconst [c] x)
+       for {
+               x := v.Args[0]
+               v_1 := v.Args[1]
+               if v_1.Op != OpAMD64MOVLconst {
+                       break
+               }
+               c := v_1.AuxInt
+               v.reset(OpAMD64TESTWconst)
+               v.AuxInt = c
+               v.AddArg(x)
+               return true
+       }
+       return false
+}
 func rewriteValueAMD64_OpAMD64XADDLlock(v *Value, config *Config) bool {
        b := v.Block
        _ = b
@@ -22237,6 +22698,132 @@ func rewriteValueAMD64_OpZeroExt8to64(v *Value, config *Config) bool {
 func rewriteBlockAMD64(b *Block, config *Config) bool {
        switch b.Kind {
        case BlockAMD64EQ:
+               // match: (EQ (TESTL (SHLL (MOVLconst [1]) x) y))
+               // cond: !config.nacl
+               // result: (UGE (BTL x y))
+               for {
+                       v := b.Control
+                       if v.Op != OpAMD64TESTL {
+                               break
+                       }
+                       v_0 := v.Args[0]
+                       if v_0.Op != OpAMD64SHLL {
+                               break
+                       }
+                       v_0_0 := v_0.Args[0]
+                       if v_0_0.Op != OpAMD64MOVLconst {
+                               break
+                       }
+                       if v_0_0.AuxInt != 1 {
+                               break
+                       }
+                       x := v_0.Args[1]
+                       y := v.Args[1]
+                       if !(!config.nacl) {
+                               break
+                       }
+                       b.Kind = BlockAMD64UGE
+                       v0 := b.NewValue0(v.Pos, OpAMD64BTL, TypeFlags)
+                       v0.AddArg(x)
+                       v0.AddArg(y)
+                       b.SetControl(v0)
+                       return true
+               }
+               // match: (EQ (TESTQ (SHLQ (MOVQconst [1]) x) y))
+               // cond: !config.nacl
+               // result: (UGE (BTQ x y))
+               for {
+                       v := b.Control
+                       if v.Op != OpAMD64TESTQ {
+                               break
+                       }
+                       v_0 := v.Args[0]
+                       if v_0.Op != OpAMD64SHLQ {
+                               break
+                       }
+                       v_0_0 := v_0.Args[0]
+                       if v_0_0.Op != OpAMD64MOVQconst {
+                               break
+                       }
+                       if v_0_0.AuxInt != 1 {
+                               break
+                       }
+                       x := v_0.Args[1]
+                       y := v.Args[1]
+                       if !(!config.nacl) {
+                               break
+                       }
+                       b.Kind = BlockAMD64UGE
+                       v0 := b.NewValue0(v.Pos, OpAMD64BTQ, TypeFlags)
+                       v0.AddArg(x)
+                       v0.AddArg(y)
+                       b.SetControl(v0)
+                       return true
+               }
+               // match: (EQ (TESTLconst [c] x))
+               // cond: isPowerOfTwo(c) && log2(c) < 32 && !config.nacl
+               // result: (UGE (BTLconst [log2(c)] x))
+               for {
+                       v := b.Control
+                       if v.Op != OpAMD64TESTLconst {
+                               break
+                       }
+                       c := v.AuxInt
+                       x := v.Args[0]
+                       if !(isPowerOfTwo(c) && log2(c) < 32 && !config.nacl) {
+                               break
+                       }
+                       b.Kind = BlockAMD64UGE
+                       v0 := b.NewValue0(v.Pos, OpAMD64BTLconst, TypeFlags)
+                       v0.AuxInt = log2(c)
+                       v0.AddArg(x)
+                       b.SetControl(v0)
+                       return true
+               }
+               // match: (EQ (TESTQconst [c] x))
+               // cond: isPowerOfTwo(c) && log2(c) < 64 && !config.nacl
+               // result: (UGE (BTQconst [log2(c)] x))
+               for {
+                       v := b.Control
+                       if v.Op != OpAMD64TESTQconst {
+                               break
+                       }
+                       c := v.AuxInt
+                       x := v.Args[0]
+                       if !(isPowerOfTwo(c) && log2(c) < 64 && !config.nacl) {
+                               break
+                       }
+                       b.Kind = BlockAMD64UGE
+                       v0 := b.NewValue0(v.Pos, OpAMD64BTQconst, TypeFlags)
+                       v0.AuxInt = log2(c)
+                       v0.AddArg(x)
+                       b.SetControl(v0)
+                       return true
+               }
+               // match: (EQ (TESTQ (MOVQconst [c]) x))
+               // cond: isPowerOfTwo(c) && log2(c) < 64 && !config.nacl
+               // result: (UGE (BTQconst [log2(c)] x))
+               for {
+                       v := b.Control
+                       if v.Op != OpAMD64TESTQ {
+                               break
+                       }
+                       v_0 := v.Args[0]
+                       if v_0.Op != OpAMD64MOVQconst {
+                               break
+                       }
+                       c := v_0.AuxInt
+                       x := v.Args[1]
+                       if !(isPowerOfTwo(c) && log2(c) < 64 && !config.nacl) {
+                               break
+                       }
+                       b.Kind = BlockAMD64UGE
+                       v0 := b.NewValue0(v.Pos, OpAMD64BTQconst, TypeFlags)
+                       v0.AuxInt = log2(c)
+                       v0.AddArg(x)
+                       b.SetControl(v0)
+                       return true
+               }
                // match: (EQ (InvertFlags cmp) yes no)
                // cond:
                // result: (EQ cmp yes no)
@@ -23278,6 +23865,132 @@ func rewriteBlockAMD64(b *Block, config *Config) bool {
                        _ = no
                        return true
                }
+               // match: (NE (TESTL (SHLL (MOVLconst [1]) x) y))
+               // cond: !config.nacl
+               // result: (ULT (BTL x y))
+               for {
+                       v := b.Control
+                       if v.Op != OpAMD64TESTL {
+                               break
+                       }
+                       v_0 := v.Args[0]
+                       if v_0.Op != OpAMD64SHLL {
+                               break
+                       }
+                       v_0_0 := v_0.Args[0]
+                       if v_0_0.Op != OpAMD64MOVLconst {
+                               break
+                       }
+                       if v_0_0.AuxInt != 1 {
+                               break
+                       }
+                       x := v_0.Args[1]
+                       y := v.Args[1]
+                       if !(!config.nacl) {
+                               break
+                       }
+                       b.Kind = BlockAMD64ULT
+                       v0 := b.NewValue0(v.Pos, OpAMD64BTL, TypeFlags)
+                       v0.AddArg(x)
+                       v0.AddArg(y)
+                       b.SetControl(v0)
+                       return true
+               }
+               // match: (NE (TESTQ (SHLQ (MOVQconst [1]) x) y))
+               // cond: !config.nacl
+               // result: (ULT (BTQ x y))
+               for {
+                       v := b.Control
+                       if v.Op != OpAMD64TESTQ {
+                               break
+                       }
+                       v_0 := v.Args[0]
+                       if v_0.Op != OpAMD64SHLQ {
+                               break
+                       }
+                       v_0_0 := v_0.Args[0]
+                       if v_0_0.Op != OpAMD64MOVQconst {
+                               break
+                       }
+                       if v_0_0.AuxInt != 1 {
+                               break
+                       }
+                       x := v_0.Args[1]
+                       y := v.Args[1]
+                       if !(!config.nacl) {
+                               break
+                       }
+                       b.Kind = BlockAMD64ULT
+                       v0 := b.NewValue0(v.Pos, OpAMD64BTQ, TypeFlags)
+                       v0.AddArg(x)
+                       v0.AddArg(y)
+                       b.SetControl(v0)
+                       return true
+               }
+               // match: (NE (TESTLconst [c] x))
+               // cond: isPowerOfTwo(c) && log2(c) < 32 && !config.nacl
+               // result: (ULT (BTLconst [log2(c)] x))
+               for {
+                       v := b.Control
+                       if v.Op != OpAMD64TESTLconst {
+                               break
+                       }
+                       c := v.AuxInt
+                       x := v.Args[0]
+                       if !(isPowerOfTwo(c) && log2(c) < 32 && !config.nacl) {
+                               break
+                       }
+                       b.Kind = BlockAMD64ULT
+                       v0 := b.NewValue0(v.Pos, OpAMD64BTLconst, TypeFlags)
+                       v0.AuxInt = log2(c)
+                       v0.AddArg(x)
+                       b.SetControl(v0)
+                       return true
+               }
+               // match: (NE (TESTQconst [c] x))
+               // cond: isPowerOfTwo(c) && log2(c) < 64 && !config.nacl
+               // result: (ULT (BTQconst [log2(c)] x))
+               for {
+                       v := b.Control
+                       if v.Op != OpAMD64TESTQconst {
+                               break
+                       }
+                       c := v.AuxInt
+                       x := v.Args[0]
+                       if !(isPowerOfTwo(c) && log2(c) < 64 && !config.nacl) {
+                               break
+                       }
+                       b.Kind = BlockAMD64ULT
+                       v0 := b.NewValue0(v.Pos, OpAMD64BTQconst, TypeFlags)
+                       v0.AuxInt = log2(c)
+                       v0.AddArg(x)
+                       b.SetControl(v0)
+                       return true
+               }
+               // match: (NE (TESTQ (MOVQconst [c]) x))
+               // cond: isPowerOfTwo(c) && log2(c) < 64 && !config.nacl
+               // result: (ULT (BTQconst [log2(c)] x))
+               for {
+                       v := b.Control
+                       if v.Op != OpAMD64TESTQ {
+                               break
+                       }
+                       v_0 := v.Args[0]
+                       if v_0.Op != OpAMD64MOVQconst {
+                               break
+                       }
+                       c := v_0.AuxInt
+                       x := v.Args[1]
+                       if !(isPowerOfTwo(c) && log2(c) < 64 && !config.nacl) {
+                               break
+                       }
+                       b.Kind = BlockAMD64ULT
+                       v0 := b.NewValue0(v.Pos, OpAMD64BTQconst, TypeFlags)
+                       v0.AuxInt = log2(c)
+                       v0.AddArg(x)
+                       b.SetControl(v0)
+                       return true
+               }
                // match: (NE (TESTB (SETGF  cmp) (SETGF  cmp)) yes no)
                // cond:
                // result: (UGT  cmp yes no)