cmd/compile: use Block.Likely to put likely branch first

When a neither of a conditional block's successors follows,
the block must end with a conditional branch followed by a
an unconditional branch.  If the (conditional) branch is
"unlikely", invert it and swap successors to make it
likely instead.

This doesn't matter to most benchmarks on amd64, but in one
instance on amd64 it caused a 30% improvement, and it is
otherwise harmless.  The problematic loop is

		for i := 0; i < w; i++ {
			if pw[i] != 0 {
				return true
			}
		}

compiled under GOEXPERIMENT=preemptibleloops
This the very worst-case benchmark for that experiment.

Also in this CL is a commoning up of heavily-repeated
boilerplate, which made it much easier to see that the
changes were applied correctly.  In the future this should
allow un-exporting of SSAGenState.Branches once the
boilerplate-replacement is done everywhere.

Change-Id: I0e5ded6eeb3ab1e3e0138e12d54c7e056bd99335
Reviewed-on: https://go-review.googlesource.com/104977
Run-TryBot: David Chase <drchase@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>

diff --git a/src/cmd/compile/internal/amd64/ssa.go b/src/cmd/compile/internal/amd64/ssa.go
index ebfe07a457bdc1d0becd27c8a74c03c1f0ad26f2..e7decb9eb6b96a4ae6161b97ca3438ac221242ae 100644 (file)
--- a/src/cmd/compile/internal/amd64/ssa.go
+++ b/src/cmd/compile/internal/amd64/ssa.go
@@ -1201,23 +1201,19 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
                ssa.BlockAMD64ULT, ssa.BlockAMD64UGT,
                ssa.BlockAMD64ULE, ssa.BlockAMD64UGE:
                jmp := blockJump[b.Kind]
-               var p *obj.Prog
                switch next {
                case b.Succs[0].Block():
-                       p = s.Prog(jmp.invasm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
+                       s.Br(jmp.invasm, b.Succs[1].Block())
                case b.Succs[1].Block():
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
+                       s.Br(jmp.asm, b.Succs[0].Block())
                default:
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
-                       q := s.Prog(obj.AJMP)
-                       q.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()})
+                       if b.Likely != ssa.BranchUnlikely {
+                               s.Br(jmp.asm, b.Succs[0].Block())
+                               s.Br(obj.AJMP, b.Succs[1].Block())
+                       } else {
+                               s.Br(jmp.invasm, b.Succs[1].Block())
+                               s.Br(obj.AJMP, b.Succs[0].Block())
+                       }
                }
 
        default:

diff --git a/src/cmd/compile/internal/arm/ssa.go b/src/cmd/compile/internal/arm/ssa.go
index abe40dfa9fe5d75684113067b036507a50152f0a..1c3b7eae11d5fd03942e8f1fa2c1175d14939c7f 100644 (file)
--- a/src/cmd/compile/internal/arm/ssa.go
+++ b/src/cmd/compile/internal/arm/ssa.go
@@ -884,23 +884,19 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
                ssa.BlockARMULT, ssa.BlockARMUGT,
                ssa.BlockARMULE, ssa.BlockARMUGE:
                jmp := blockJump[b.Kind]
-               var p *obj.Prog
                switch next {
                case b.Succs[0].Block():
-                       p = s.Prog(jmp.invasm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
+                       s.Br(jmp.invasm, b.Succs[1].Block())
                case b.Succs[1].Block():
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
+                       s.Br(jmp.asm, b.Succs[0].Block())
                default:
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
-                       q := s.Prog(obj.AJMP)
-                       q.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()})
+                       if b.Likely != ssa.BranchUnlikely {
+                               s.Br(jmp.asm, b.Succs[0].Block())
+                               s.Br(obj.AJMP, b.Succs[1].Block())
+                       } else {
+                               s.Br(jmp.invasm, b.Succs[1].Block())
+                               s.Br(obj.AJMP, b.Succs[0].Block())
+                       }
                }
 
        default:

diff --git a/src/cmd/compile/internal/arm64/ssa.go b/src/cmd/compile/internal/arm64/ssa.go
index b72ead73686dae50192a36a2488b5d2439639ad6..11e7002df401cd72bc32c6668016d6019adc01de 100644 (file)
--- a/src/cmd/compile/internal/arm64/ssa.go
+++ b/src/cmd/compile/internal/arm64/ssa.go
@@ -874,20 +874,17 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
                var p *obj.Prog
                switch next {
                case b.Succs[0].Block():
-                       p = s.Prog(jmp.invasm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
+                       p = s.Br(jmp.invasm, b.Succs[1].Block())
                case b.Succs[1].Block():
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
+                       p = s.Br(jmp.asm, b.Succs[0].Block())
                default:
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
-                       q := s.Prog(obj.AJMP)
-                       q.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()})
+                       if b.Likely != ssa.BranchUnlikely {
+                               p = s.Br(jmp.asm, b.Succs[0].Block())
+                               s.Br(obj.AJMP, b.Succs[1].Block())
+                       } else {
+                               p = s.Br(jmp.invasm, b.Succs[1].Block())
+                               s.Br(obj.AJMP, b.Succs[0].Block())
+                       }
                }
                if !b.Control.Type.IsFlags() {
                        p.From.Type = obj.TYPE_REG
@@ -898,30 +895,21 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
                var p *obj.Prog
                switch next {
                case b.Succs[0].Block():
-                       p = s.Prog(jmp.invasm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       p.From.Offset = b.Aux.(int64)
-                       p.From.Type = obj.TYPE_CONST
-                       p.Reg = b.Control.Reg()
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
+                       p = s.Br(jmp.invasm, b.Succs[1].Block())
                case b.Succs[1].Block():
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       p.From.Offset = b.Aux.(int64)
-                       p.From.Type = obj.TYPE_CONST
-                       p.Reg = b.Control.Reg()
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
+                       p = s.Br(jmp.asm, b.Succs[0].Block())
                default:
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       p.From.Offset = b.Aux.(int64)
-                       p.From.Type = obj.TYPE_CONST
-                       p.Reg = b.Control.Reg()
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
-                       q := s.Prog(obj.AJMP)
-                       q.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()})
+                       if b.Likely != ssa.BranchUnlikely {
+                               p = s.Br(jmp.asm, b.Succs[0].Block())
+                               s.Br(obj.AJMP, b.Succs[1].Block())
+                       } else {
+                               p = s.Br(jmp.invasm, b.Succs[1].Block())
+                               s.Br(obj.AJMP, b.Succs[0].Block())
+                       }
                }
+               p.From.Offset = b.Aux.(int64)
+               p.From.Type = obj.TYPE_CONST
+               p.Reg = b.Control.Reg()
 
        default:
                b.Fatalf("branch not implemented: %s. Control: %s", b.LongString(), b.Control.LongString())

diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go
index 177e0aaafb4f4dea72de08b4833bad7845711c64..69c20719ce37d2d456beb0c928cf358e01c946e7 100644 (file)
--- a/src/cmd/compile/internal/gc/ssa.go
+++ b/src/cmd/compile/internal/gc/ssa.go
@@ -4684,6 +4684,16 @@ func (s *SSAGenState) SetPos(pos src.XPos) {
        s.pp.pos = pos
 }
 
+// Br emits a single branch instruction and returns the instruction.
+// Not all architectures need the returned instruction, but otherwise
+// the boilerplate is common to all.
+func (s *SSAGenState) Br(op obj.As, target *ssa.Block) *obj.Prog {
+       p := s.Prog(op)
+       p.To.Type = obj.TYPE_BRANCH
+       s.Branches = append(s.Branches, Branch{P: p, B: target})
+       return p
+}
+
 // DebugFriendlySetPos sets the position subject to heuristics
 // that reduce "jumpy" line number churn when debugging.
 // Spill/fill/copy instructions from the register allocator,

diff --git a/src/cmd/compile/internal/mips/ssa.go b/src/cmd/compile/internal/mips/ssa.go
index 61dedb00caa78e23a0f7c25323772125f3562a59..0098d1ce2b55f8fb28a930cf95406569810002c8 100644 (file)
--- a/src/cmd/compile/internal/mips/ssa.go
+++ b/src/cmd/compile/internal/mips/ssa.go
@@ -828,20 +828,17 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
                var p *obj.Prog
                switch next {
                case b.Succs[0].Block():
-                       p = s.Prog(jmp.invasm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
+                       p = s.Br(jmp.invasm, b.Succs[1].Block())
                case b.Succs[1].Block():
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
+                       p = s.Br(jmp.asm, b.Succs[0].Block())
                default:
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
-                       q := s.Prog(obj.AJMP)
-                       q.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()})
+                       if b.Likely != ssa.BranchUnlikely {
+                               p = s.Br(jmp.asm, b.Succs[0].Block())
+                               s.Br(obj.AJMP, b.Succs[1].Block())
+                       } else {
+                               p = s.Br(jmp.invasm, b.Succs[1].Block())
+                               s.Br(obj.AJMP, b.Succs[0].Block())
+                       }
                }
                if !b.Control.Type.IsFlags() {
                        p.From.Type = obj.TYPE_REG

diff --git a/src/cmd/compile/internal/mips64/ssa.go b/src/cmd/compile/internal/mips64/ssa.go
index 8f35fd039ac5a70160a9aedc9ceeb28fd51e5389..d8645946de430dd75109894831aefae40783f5a6 100644 (file)
--- a/src/cmd/compile/internal/mips64/ssa.go
+++ b/src/cmd/compile/internal/mips64/ssa.go
@@ -799,20 +799,17 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
                var p *obj.Prog
                switch next {
                case b.Succs[0].Block():
-                       p = s.Prog(jmp.invasm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
+                       p = s.Br(jmp.invasm, b.Succs[1].Block())
                case b.Succs[1].Block():
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
+                       p = s.Br(jmp.asm, b.Succs[0].Block())
                default:
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
-                       q := s.Prog(obj.AJMP)
-                       q.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()})
+                       if b.Likely != ssa.BranchUnlikely {
+                               p = s.Br(jmp.asm, b.Succs[0].Block())
+                               s.Br(obj.AJMP, b.Succs[1].Block())
+                       } else {
+                               p = s.Br(jmp.invasm, b.Succs[1].Block())
+                               s.Br(obj.AJMP, b.Succs[0].Block())
+                       }
                }
                if !b.Control.Type.IsFlags() {
                        p.From.Type = obj.TYPE_REG

diff --git a/src/cmd/compile/internal/ppc64/ssa.go b/src/cmd/compile/internal/ppc64/ssa.go
index 8d843f07567d7ea8d69c2af3c7d81742b52116d1..7e470e55b992d0973cff15766e7dce3fa430ff96 100644 (file)
--- a/src/cmd/compile/internal/ppc64/ssa.go
+++ b/src/cmd/compile/internal/ppc64/ssa.go
@@ -1176,41 +1176,34 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
                ssa.BlockPPC64FLT, ssa.BlockPPC64FGE,
                ssa.BlockPPC64FLE, ssa.BlockPPC64FGT:
                jmp := blockJump[b.Kind]
-               var p *obj.Prog
                switch next {
                case b.Succs[0].Block():
-                       p = s.Prog(jmp.invasm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
+                       s.Br(jmp.invasm, b.Succs[1].Block())
                        if jmp.invasmun {
                                // TODO: The second branch is probably predict-not-taken since it is for FP unordered
-                               q := s.Prog(ppc64.ABVS)
-                               q.To.Type = obj.TYPE_BRANCH
-                               s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()})
+                               s.Br(ppc64.ABVS, b.Succs[1].Block())
                        }
                case b.Succs[1].Block():
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
+                       s.Br(jmp.asm, b.Succs[0].Block())
                        if jmp.asmeq {
-                               q := s.Prog(ppc64.ABEQ)
-                               q.To.Type = obj.TYPE_BRANCH
-                               s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[0].Block()})
+                               s.Br(ppc64.ABEQ, b.Succs[0].Block())
                        }
                default:
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
-                       if jmp.asmeq {
-                               q := s.Prog(ppc64.ABEQ)
-                               q.To.Type = obj.TYPE_BRANCH
-                               s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[0].Block()})
+                       if b.Likely != ssa.BranchUnlikely {
+                               s.Br(jmp.asm, b.Succs[0].Block())
+                               if jmp.asmeq {
+                                       s.Br(ppc64.ABEQ, b.Succs[0].Block())
+                               }
+                               s.Br(obj.AJMP, b.Succs[1].Block())
+                       } else {
+                               s.Br(jmp.invasm, b.Succs[1].Block())
+                               if jmp.invasmun {
+                                       // TODO: The second branch is probably predict-not-taken since it is for FP unordered
+                                       s.Br(ppc64.ABVS, b.Succs[1].Block())
+                               }
+                               s.Br(obj.AJMP, b.Succs[0].Block())
                        }
-                       q := s.Prog(obj.AJMP)
-                       q.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()})
                }
-
        default:
                b.Fatalf("branch not implemented: %s. Control: %s", b.LongString(), b.Control.LongString())
        }

diff --git a/src/cmd/compile/internal/s390x/ssa.go b/src/cmd/compile/internal/s390x/ssa.go
index 23735ec3a6899be82a7d630658cec4c0faf9993e..961bef5b910aead6811aee6633026d2369cd253c 100644 (file)
--- a/src/cmd/compile/internal/s390x/ssa.go
+++ b/src/cmd/compile/internal/s390x/ssa.go
@@ -821,23 +821,19 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
                ssa.BlockS390XLE, ssa.BlockS390XGT,
                ssa.BlockS390XGEF, ssa.BlockS390XGTF:
                jmp := blockJump[b.Kind]
-               var p *obj.Prog
                switch next {
                case b.Succs[0].Block():
-                       p = s.Prog(jmp.invasm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
+                       s.Br(jmp.invasm, b.Succs[1].Block())
                case b.Succs[1].Block():
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
+                       s.Br(jmp.asm, b.Succs[0].Block())
                default:
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
-                       q := s.Prog(s390x.ABR)
-                       q.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()})
+                       if b.Likely != ssa.BranchUnlikely {
+                               s.Br(jmp.asm, b.Succs[0].Block())
+                               s.Br(s390x.ABR, b.Succs[1].Block())
+                       } else {
+                               s.Br(jmp.invasm, b.Succs[1].Block())
+                               s.Br(s390x.ABR, b.Succs[0].Block())
+                       }
                }
        default:
                b.Fatalf("branch not implemented: %s. Control: %s", b.LongString(), b.Control.LongString())

diff --git a/src/cmd/compile/internal/x86/ssa.go b/src/cmd/compile/internal/x86/ssa.go
index 91cf70ff89219f3428a646388287db32a0b008cb..1e0e1f9a702949bfcb908453592fc9f8089d5734 100644 (file)
--- a/src/cmd/compile/internal/x86/ssa.go
+++ b/src/cmd/compile/internal/x86/ssa.go
@@ -863,25 +863,20 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
                ssa.Block386ULT, ssa.Block386UGT,
                ssa.Block386ULE, ssa.Block386UGE:
                jmp := blockJump[b.Kind]
-               var p *obj.Prog
                switch next {
                case b.Succs[0].Block():
-                       p = s.Prog(jmp.invasm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
+                       s.Br(jmp.invasm, b.Succs[1].Block())
                case b.Succs[1].Block():
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
+                       s.Br(jmp.asm, b.Succs[0].Block())
                default:
-                       p = s.Prog(jmp.asm)
-                       p.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
-                       q := s.Prog(obj.AJMP)
-                       q.To.Type = obj.TYPE_BRANCH
-                       s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()})
+                       if b.Likely != ssa.BranchUnlikely {
+                               s.Br(jmp.asm, b.Succs[0].Block())
+                               s.Br(obj.AJMP, b.Succs[1].Block())
+                       } else {
+                               s.Br(jmp.invasm, b.Succs[1].Block())
+                               s.Br(obj.AJMP, b.Succs[0].Block())
+                       }
                }
-
        default:
                b.Fatalf("branch not implemented: %s. Control: %s", b.LongString(), b.Control.LongString())
        }