From: Austin Clements Date: Tue, 27 Feb 2018 01:56:58 +0000 (-0500) Subject: cmd/compile: compute register liveness maps X-Git-Tag: go1.11beta1~365 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=61158c162fa19dbe73d23e6e094ae310adf8918b;p=gostls13.git cmd/compile: compute register liveness maps This extends the liveness analysis to track registers containing live pointers. We do this by tracking bitmaps for live pointer registers in parallel with bitmaps for stack variables. This does not yet do anything with these liveness maps, though they do appear in the debug output for -live=2. We'll optimize this in later CLs: name old time/op new time/op delta Template 193ms ± 5% 195ms ± 2% ~ (p=0.050 n=9+9) Unicode 97.7ms ± 2% 98.4ms ± 2% ~ (p=0.315 n=9+10) GoTypes 674ms ± 2% 685ms ± 1% +1.72% (p=0.001 n=9+9) Compiler 3.21s ± 1% 3.28s ± 1% +2.28% (p=0.000 n=10+9) SSA 7.70s ± 1% 7.79s ± 1% +1.07% (p=0.015 n=10+10) Flate 130ms ± 3% 133ms ± 2% +2.19% (p=0.003 n=10+10) GoParser 159ms ± 3% 161ms ± 2% +1.51% (p=0.019 n=10+10) Reflect 444ms ± 1% 450ms ± 1% +1.43% (p=0.000 n=9+10) Tar 181ms ± 2% 183ms ± 2% +1.45% (p=0.010 n=10+9) XML 230ms ± 1% 234ms ± 1% +1.56% (p=0.000 n=8+9) [Geo mean] 405ms 411ms +1.48% No effect on binary size because we're not yet emitting the register maps. For #24543. Change-Id: Ieb022f0aea89c0ea9a6f035195bce2f0e67dbae4 Reviewed-on: https://go-review.googlesource.com/109352 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- diff --git a/src/cmd/compile/fmt_test.go b/src/cmd/compile/fmt_test.go index d224e67ea4..9d47b04b1c 100644 --- a/src/cmd/compile/fmt_test.go +++ b/src/cmd/compile/fmt_test.go @@ -642,8 +642,10 @@ var knownFormats = map[string]string{ "cmd/compile/internal/ssa.GCNode %v": "", "cmd/compile/internal/ssa.ID %d": "", "cmd/compile/internal/ssa.ID %v": "", + "cmd/compile/internal/ssa.LocPair %s": "", "cmd/compile/internal/ssa.LocalSlot %s": "", "cmd/compile/internal/ssa.LocalSlot %v": "", + "cmd/compile/internal/ssa.Location %T": "", "cmd/compile/internal/ssa.Location %s": "", "cmd/compile/internal/ssa.Op %s": "", "cmd/compile/internal/ssa.Op %v": "", diff --git a/src/cmd/compile/internal/gc/plive.go b/src/cmd/compile/internal/gc/plive.go index cb5eecf430..9da398a459 100644 --- a/src/cmd/compile/internal/gc/plive.go +++ b/src/cmd/compile/internal/gc/plive.go @@ -88,8 +88,8 @@ type BlockEffects struct { // uevar: upward exposed variables (used before set in block) // varkill: killed variables (set in block) // avarinit: addrtaken variables set or used (proof of initialization) - uevar bvec - varkill bvec + uevar varRegVec + varkill varRegVec avarinit bvec // Computed during Liveness.solve using control flow information: @@ -100,8 +100,8 @@ type BlockEffects struct { // (initialized in block or at exit from any predecessor block) // avarinitall: addrtaken variables certainly initialized at block exit // (initialized in block or at exit from all predecessor blocks) - livein bvec - liveout bvec + livein varRegVec + liveout varRegVec avarinitany bvec avarinitall bvec } @@ -121,7 +121,7 @@ type Liveness struct { // An array with a bit vector for each safe point tracking live variables. // Indexed sequentially by safe points in Block and Value order. - livevars []bvec + livevars []varRegVec // livenessMap maps from safe points (i.e., CALLs) to their // liveness map indexes. @@ -130,6 +130,7 @@ type Liveness struct { // should this be a dense structure? livenessMap LivenessMap stackMaps []bvec + regMaps []liveRegMask cache progeffectscache } @@ -150,6 +151,7 @@ func (m LivenessMap) Get(v *ssa.Value) LivenessIndex { // LivenessIndex stores the liveness map index for a safe-point. type LivenessIndex struct { stackMapIndex int + regMapIndex int } // LivenessInvalid indicates an unsafe point. @@ -159,7 +161,7 @@ type LivenessIndex struct { // index 0; sigh). TODO(austin): Maybe we should use PCDATA+1 as the // index into the liveness map so -1 uniquely refers to the entry // liveness map. -var LivenessInvalid = LivenessIndex{-2} +var LivenessInvalid = LivenessIndex{-2, -2} func (idx LivenessIndex) Valid() bool { return idx.stackMapIndex >= 0 @@ -172,6 +174,36 @@ type progeffectscache struct { initialized bool } +// varRegVec contains liveness bitmaps for variables and registers. +type varRegVec struct { + vars bvec + regs liveRegMask +} + +func (v *varRegVec) Eq(v2 varRegVec) bool { + return v.vars.Eq(v2.vars) && v.regs == v2.regs +} + +func (v *varRegVec) Copy(v2 varRegVec) { + v.vars.Copy(v2.vars) + v.regs = v2.regs +} + +func (v *varRegVec) Clear() { + v.vars.Clear() + v.regs = 0 +} + +func (v *varRegVec) Or(v1, v2 varRegVec) { + v.vars.Or(v1.vars, v2.vars) + v.regs = v1.regs | v2.regs +} + +func (v *varRegVec) AndNot(v1, v2 varRegVec) { + v.vars.AndNot(v1.vars, v2.vars) + v.regs = v1.regs &^ v2.regs +} + // livenessShouldTrack reports whether the liveness analysis // should track the variable n. // We don't care about variables that have no pointers, @@ -349,6 +381,107 @@ func affectedNode(v *ssa.Value) (*Node, ssa.SymEffect) { } } +// regEffects returns the registers affected by v. +func (lv *Liveness) regEffects(v *ssa.Value) (uevar, kill liveRegMask) { + if v.Op == ssa.OpPhi { + // All phi node arguments must come from the same + // register and the result must also go to that + // register, so there's no overall effect. + return 0, 0 + } + addLocs := func(mask liveRegMask, v *ssa.Value, ptrOnly bool) liveRegMask { + if int(v.ID) >= len(lv.f.RegAlloc) { + // v has no allocated registers. + return mask + } + loc := lv.f.RegAlloc[v.ID] + if loc == nil { + // v has no allocated registers. + return mask + } + if v.Op == ssa.OpGetG { + // GetG represents the G register, which is a + // pointer, but not a valid GC register. The + // current G is always reachable, so it's okay + // to ignore this register. + return mask + } + + // Collect registers and types from v's location. + var regs [2]*ssa.Register + nreg := 0 + switch loc := loc.(type) { + case ssa.LocalSlot: + return mask + case *ssa.Register: + if ptrOnly && !v.Type.HasHeapPointer() { + return mask + } + regs[0] = loc + nreg = 1 + case ssa.LocPair: + // The value will have TTUPLE type, and the + // children are nil or *ssa.Register. + if v.Type.Etype != types.TTUPLE { + v.Fatalf("location pair %s has non-tuple type %v", loc, v.Type) + } + for i, loc1 := range loc { + if loc1 == nil { + continue + } + if ptrOnly && !v.Type.FieldType(i).HasHeapPointer() { + continue + } + regs[nreg] = loc1.(*ssa.Register) + nreg++ + } + default: + v.Fatalf("weird RegAlloc location: %s (%T)", loc, loc) + } + + // Add register locations to vars. + for _, reg := range regs[:nreg] { + if reg.GCNum() == -1 { + if ptrOnly { + v.Fatalf("pointer in non-pointer register %v", reg) + } else { + continue + } + } + mask |= 1 << uint(reg.GCNum()) + } + return mask + } + + // v clobbers all registers it writes to (whether or not the + // write is pointer-typed). + kill = addLocs(0, v, false) + for _, arg := range v.Args { + // v uses all registers is reads from, but we only + // care about marking those containing pointers. + uevar = addLocs(uevar, arg, true) + } + return uevar, kill +} + +type liveRegMask uint32 + +func (m liveRegMask) niceString(config *ssa.Config) string { + if m == 0 { + return "" + } + str := "" + for i, reg := range config.GCRegMap { + if m&(1<= 0; j-- { pos, e := lv.valueEffects(b.Values[j]) + regUevar, regKill := lv.regEffects(b.Values[j]) if e&varkill != 0 { - be.varkill.Set(pos) - be.uevar.Unset(pos) + be.varkill.vars.Set(pos) + be.uevar.vars.Unset(pos) } + be.varkill.regs |= regKill + be.uevar.regs &^= regKill if e&uevar != 0 { - be.uevar.Set(pos) + be.uevar.vars.Set(pos) } + be.uevar.regs |= regUevar } // Walk the block instructions forward to update avarinit bits. // avarinit describes the effect at the end of the block, not the beginning. for _, val := range b.Values { pos, e := lv.valueEffects(val) + // No need for regEffects because registers never appear in avarinit. if e&varkill != 0 { be.avarinit.Unset(pos) } @@ -691,10 +829,11 @@ func (lv *Liveness) prologue() { func (lv *Liveness) solve() { // These temporary bitvectors exist to avoid successive allocations and // frees within the loop. - newlivein := bvalloc(int32(len(lv.vars))) - newliveout := bvalloc(int32(len(lv.vars))) - any := bvalloc(int32(len(lv.vars))) - all := bvalloc(int32(len(lv.vars))) + nvars := int32(len(lv.vars)) + newlivein := varRegVec{vars: bvalloc(nvars)} + newliveout := varRegVec{vars: bvalloc(nvars)} + any := bvalloc(nvars) + all := bvalloc(nvars) // Push avarinitall, avarinitany forward. // avarinitall says the addressed var is initialized along all paths reaching the block exit. @@ -722,8 +861,8 @@ func (lv *Liveness) solve() { be := lv.blockEffects(b) lv.avarinitanyall(b, any, all) - any.AndNot(any, be.varkill) - all.AndNot(all, be.varkill) + any.AndNot(any, be.varkill.vars) + all.AndNot(all, be.varkill.vars) any.Or(any, be.avarinit) all.Or(all, be.avarinit) if !any.Eq(be.avarinitany) { @@ -751,11 +890,11 @@ func (lv *Liveness) solve() { switch b.Kind { case ssa.BlockRet: for _, pos := range lv.cache.retuevar { - newliveout.Set(pos) + newliveout.vars.Set(pos) } case ssa.BlockRetJmp: for _, pos := range lv.cache.tailuevar { - newliveout.Set(pos) + newliveout.vars.Set(pos) } case ssa.BlockExit: // nothing to do @@ -790,7 +929,7 @@ func (lv *Liveness) solve() { // variables at each safe point locations. func (lv *Liveness) epilogue() { nvars := int32(len(lv.vars)) - liveout := bvalloc(nvars) + liveout := varRegVec{vars: bvalloc(nvars)} any := bvalloc(nvars) all := bvalloc(nvars) livedefer := bvalloc(nvars) // always-live variables @@ -830,7 +969,7 @@ func (lv *Liveness) epilogue() { for _, pos := range lv.cache.textavarinit { live.Set(pos) } - lv.livevars = append(lv.livevars, live) + lv.livevars = append(lv.livevars, varRegVec{vars: live}) } for _, b := range lv.f.Blocks { @@ -846,6 +985,7 @@ func (lv *Liveness) epilogue() { // Seed the maps with information about the addrtaken variables. for _, v := range b.Values { pos, e := lv.valueEffects(v) + // No need for regEffects because registers never appear in avarinit. if e&varkill != 0 { any.Unset(pos) all.Unset(pos) @@ -862,10 +1002,10 @@ func (lv *Liveness) epilogue() { // Annotate ambiguously live variables so that they can // be zeroed at function entry and at VARKILL points. // liveout is dead here and used as a temporary. - liveout.AndNot(any, all) - if !liveout.IsEmpty() { - for pos := int32(0); pos < liveout.n; pos++ { - if !liveout.Get(pos) { + liveout.vars.AndNot(any, all) + if !liveout.vars.IsEmpty() { + for pos := int32(0); pos < liveout.vars.n; pos++ { + if !liveout.vars.Get(pos) { continue } all.Set(pos) // silence future warnings in this block @@ -882,7 +1022,7 @@ func (lv *Liveness) epilogue() { // Live stuff first. live := bvalloc(nvars) live.Copy(any) - lv.livevars = append(lv.livevars, live) + lv.livevars = append(lv.livevars, varRegVec{vars: live}) } be.lastbitmapindex = len(lv.livevars) - 1 @@ -907,20 +1047,23 @@ func (lv *Liveness) epilogue() { // Found an interesting instruction, record the // corresponding liveness information. - live := lv.livevars[index] - live.Or(live, liveout) - live.Or(live, livedefer) // only for non-entry safe points + live := &lv.livevars[index] + live.Or(*live, liveout) + live.vars.Or(live.vars, livedefer) // only for non-entry safe points index-- } // Update liveness information. pos, e := lv.valueEffects(v) + regUevar, regKill := lv.regEffects(v) if e&varkill != 0 { - liveout.Unset(pos) + liveout.vars.Unset(pos) } + liveout.regs &^= regKill if e&uevar != 0 { - liveout.Set(pos) + liveout.vars.Set(pos) } + liveout.regs |= regUevar } if b == lv.f.Entry { @@ -929,8 +1072,8 @@ func (lv *Liveness) epilogue() { } // Record live variables. - live := lv.livevars[index] - live.Or(live, liveout) + live := &lv.livevars[index] + live.Or(*live, liveout) } } @@ -938,10 +1081,35 @@ func (lv *Liveness) epilogue() { // the only things that can possibly be live are the // input parameters. for j, n := range lv.vars { - if n.Class() != PPARAM && lv.livevars[0].Get(int32(j)) { + if n.Class() != PPARAM && lv.livevars[0].vars.Get(int32(j)) { Fatalf("internal error: %v %L recorded as live on entry", lv.fn.Func.Nname, n) } } + // Check that no registers are live at function entry. + // The context register, if any, comes from a + // LoweredGetClosurePtr operation first thing in the function, + // so it doesn't appear live at entry. + if regs := lv.livevars[0].regs; regs != 0 { + lv.printDebug() + lv.f.Fatalf("internal error: %v register %s recorded as live on entry", lv.fn.Func.Nname, regs.niceString(lv.f.Config)) + } + // Check that no registers are live across calls. + // For closure calls, the CALLclosure is the last use + // of the context register, so it's dead after the call. + for _, b := range lv.f.Blocks { + index := int32(lv.blockEffects(b).lastbitmapindex) + for i := len(b.Values) - 1; i >= 0; i-- { + v := b.Values[i] + if lv.issafepoint(v) { + live := lv.livevars[index] + if v.Op.IsCall() && live.regs != 0 { + lv.printDebug() + v.Fatalf("internal error: %v register %s recorded as live at call", lv.fn.Func.Nname, live.regs.niceString(lv.f.Config)) + } + index-- + } + } + } } func (lv *Liveness) clobber() { @@ -1177,7 +1345,7 @@ func (lv *Liveness) compact() { // If already seen, record earlier index in remap. Outer: for i, live := range lv.livevars { - h := hashbitmap(H0, live) % uint32(tablesize) + h := hashbitmap(H0, live.vars) % uint32(tablesize) for { j := table[h] @@ -1185,7 +1353,7 @@ Outer: break } jlive := lv.stackMaps[j] - if live.Eq(jlive) { + if live.vars.Eq(jlive) { remap[i] = j continue Outer } @@ -1198,7 +1366,20 @@ Outer: table[h] = len(lv.stackMaps) remap[i] = len(lv.stackMaps) - lv.stackMaps = append(lv.stackMaps, live) + lv.stackMaps = append(lv.stackMaps, live.vars) + } + + // Compact register maps. + remapRegs := make([]int, len(lv.livevars)) + regMaps := make(map[liveRegMask]int) + for i, live := range lv.livevars { + idx, ok := regMaps[live.regs] + if !ok { + idx = len(regMaps) + regMaps[live.regs] = idx + lv.regMaps = append(lv.regMaps, live.regs) + } + remapRegs[i] = idx } // Clear lv.livevars to allow GC of duplicate maps and to @@ -1214,7 +1395,7 @@ Outer: for _, v := range b.Values { if lv.issafepoint(v) { lv.showlive(v, lv.stackMaps[remap[pos]]) - lv.livenessMap.m[v] = LivenessIndex{remap[pos]} + lv.livenessMap.m[v] = LivenessIndex{remap[pos], remapRegs[pos]} pos++ } } @@ -1261,33 +1442,33 @@ func (lv *Liveness) showlive(v *ssa.Value, live bvec) { Warnl(pos, s) } -func (lv *Liveness) printbvec(printed bool, name string, live bvec) bool { - started := false +func (lv *Liveness) printbvec(printed bool, name string, live varRegVec) bool { + if live.vars.IsEmpty() && live.regs == 0 { + return printed + } + + if !printed { + fmt.Printf("\t") + } else { + fmt.Printf(" ") + } + fmt.Printf("%s=", name) + + comma := "" for i, n := range lv.vars { - if !live.Get(int32(i)) { + if !live.vars.Get(int32(i)) { continue } - if !started { - if !printed { - fmt.Printf("\t") - } else { - fmt.Printf(" ") - } - started = true - printed = true - fmt.Printf("%s=", name) - } else { - fmt.Printf(",") - } - - fmt.Printf("%s", n.Sym.Name) + fmt.Printf("%s%s", comma, n.Sym.Name) + comma = "," } - return printed + fmt.Printf("%s%s", comma, live.regs.niceString(lv.f.Config)) + return true } -// printeffect is like printbvec, but for a single variable. -func (lv *Liveness) printeffect(printed bool, name string, pos int32, x bool) bool { - if !x { +// printeffect is like printbvec, but for valueEffects and regEffects. +func (lv *Liveness) printeffect(printed bool, name string, pos int32, x bool, regMask liveRegMask) bool { + if !x && regMask == 0 { return printed } if !printed { @@ -1295,7 +1476,19 @@ func (lv *Liveness) printeffect(printed bool, name string, pos int32, x bool) bo } else { fmt.Printf(" ") } - fmt.Printf("%s=%s", name, lv.vars[pos].Sym.Name) + fmt.Printf("%s=", name) + if x { + fmt.Printf("%s", lv.vars[pos].Sym.Name) + } + for j, reg := range lv.f.Config.GCRegMap { + if regMask&(1<