"strings"
)
-// go115ReduceLiveness disables register maps and only produces stack
-// maps at call sites.
-//
-// In Go 1.15, we changed debug call injection to use conservative
-// scanning instead of precise pointer maps, so these are no longer
-// necessary.
-//
-// Keep in sync with runtime/preempt.go:go115ReduceLiveness.
-const go115ReduceLiveness = true
-
// OpVarDef is an annotation for the liveness analysis, marking a place
// where a complete initialization (definition) of a variable begins.
// Since the liveness analysis can see initialization of single-word
//
// uevar: upward exposed variables (used before set in block)
// varkill: killed variables (set in block)
- uevar varRegVec
- varkill varRegVec
+ uevar bvec
+ varkill bvec
// Computed during Liveness.solve using control flow information:
//
// livein: variables live at block entry
// liveout: variables live at block exit
- livein varRegVec
- liveout varRegVec
+ livein bvec
+ liveout bvec
}
// A collection of global state used by liveness analysis.
// current Block during Liveness.epilogue. Indexed in Value
// order for that block. Additionally, for the entry block
// livevars[0] is the entry bitmap. Liveness.compact moves
- // these to stackMaps and regMaps.
- livevars []varRegVec
+ // these to stackMaps.
+ livevars []bvec
// livenessMap maps from safe points (i.e., CALLs) to their
// liveness map indexes.
livenessMap LivenessMap
stackMapSet bvecSet
stackMaps []bvec
- regMapSet map[liveRegMask]int
- regMaps []liveRegMask
cache progeffectscache
}
delete(m.vals, k)
}
}
- m.deferreturn = LivenessInvalid
+ m.deferreturn = LivenessDontCare
}
func (m *LivenessMap) set(v *ssa.Value, i LivenessIndex) {
}
func (m LivenessMap) Get(v *ssa.Value) LivenessIndex {
- if !go115ReduceLiveness {
- // All safe-points are in the map, so if v isn't in
- // the map, it's an unsafe-point.
- if idx, ok := m.vals[v.ID]; ok {
- return idx
- }
- return LivenessInvalid
- }
-
// If v isn't in the map, then it's a "don't care" and not an
// unsafe-point.
if idx, ok := m.vals[v.ID]; ok {
return idx
}
- return LivenessIndex{StackMapDontCare, StackMapDontCare, false}
+ return LivenessIndex{StackMapDontCare, false}
}
// LivenessIndex stores the liveness map information for a Value.
type LivenessIndex struct {
stackMapIndex int
- regMapIndex int // only for !go115ReduceLiveness
// isUnsafePoint indicates that this is an unsafe-point.
//
isUnsafePoint bool
}
-// LivenessInvalid indicates an unsafe point with no stack map.
-var LivenessInvalid = LivenessIndex{StackMapDontCare, StackMapDontCare, true} // only for !go115ReduceLiveness
+// LivenessDontCare indicates that the liveness information doesn't
+// matter. Currently it is used in deferreturn liveness when we don't
+// actually need it. It should never be emitted to the PCDATA stream.
+var LivenessDontCare = LivenessIndex{StackMapDontCare, true}
// StackMapDontCare indicates that the stack map index at a Value
// doesn't matter.
return idx.stackMapIndex != StackMapDontCare
}
-func (idx LivenessIndex) RegMapValid() bool {
- return idx.regMapIndex != StackMapDontCare
-}
-
type progeffectscache struct {
retuevar []int32
tailuevar []int32
initialized bool
}
-// varRegVec contains liveness bitmaps for variables and registers.
-type varRegVec struct {
- vars bvec
- regs liveRegMask
-}
-
-func (v *varRegVec) Eq(v2 varRegVec) bool {
- return v.vars.Eq(v2.vars) && v.regs == v2.regs
-}
-
-func (v *varRegVec) Copy(v2 varRegVec) {
- v.vars.Copy(v2.vars)
- v.regs = v2.regs
-}
-
-func (v *varRegVec) Clear() {
- v.vars.Clear()
- v.regs = 0
-}
-
-func (v *varRegVec) Or(v1, v2 varRegVec) {
- v.vars.Or(v1.vars, v2.vars)
- v.regs = v1.regs | v2.regs
-}
-
-func (v *varRegVec) AndNot(v1, v2 varRegVec) {
- v.vars.AndNot(v1.vars, v2.vars)
- v.regs = v1.regs &^ v2.regs
-}
-
// livenessShouldTrack reports whether the liveness analysis
// should track the variable n.
// We don't care about variables that have no pointers,
}
}
-// regEffects returns the registers affected by v.
-func (lv *Liveness) regEffects(v *ssa.Value) (uevar, kill liveRegMask) {
- if go115ReduceLiveness {
- return 0, 0
- }
- if v.Op == ssa.OpPhi {
- // All phi node arguments must come from the same
- // register and the result must also go to that
- // register, so there's no overall effect.
- return 0, 0
- }
- addLocs := func(mask liveRegMask, v *ssa.Value, ptrOnly bool) liveRegMask {
- if int(v.ID) >= len(lv.f.RegAlloc) {
- // v has no allocated registers.
- return mask
- }
- loc := lv.f.RegAlloc[v.ID]
- if loc == nil {
- // v has no allocated registers.
- return mask
- }
- if v.Op == ssa.OpGetG {
- // GetG represents the G register, which is a
- // pointer, but not a valid GC register. The
- // current G is always reachable, so it's okay
- // to ignore this register.
- return mask
- }
-
- // Collect registers and types from v's location.
- var regs [2]*ssa.Register
- nreg := 0
- switch loc := loc.(type) {
- case ssa.LocalSlot:
- return mask
- case *ssa.Register:
- if ptrOnly && !v.Type.HasPointers() {
- return mask
- }
- regs[0] = loc
- nreg = 1
- case ssa.LocPair:
- // The value will have TTUPLE type, and the
- // children are nil or *ssa.Register.
- if v.Type.Etype != types.TTUPLE {
- v.Fatalf("location pair %s has non-tuple type %v", loc, v.Type)
- }
- for i, loc1 := range &loc {
- if loc1 == nil {
- continue
- }
- if ptrOnly && !v.Type.FieldType(i).HasPointers() {
- continue
- }
- regs[nreg] = loc1.(*ssa.Register)
- nreg++
- }
- default:
- v.Fatalf("weird RegAlloc location: %s (%T)", loc, loc)
- }
-
- // Add register locations to vars.
- for _, reg := range regs[:nreg] {
- if reg.GCNum() == -1 {
- if ptrOnly {
- v.Fatalf("pointer in non-pointer register %v", reg)
- } else {
- continue
- }
- }
- mask |= 1 << uint(reg.GCNum())
- }
- return mask
- }
-
- // v clobbers all registers it writes to (whether or not the
- // write is pointer-typed).
- kill = addLocs(0, v, false)
- for _, arg := range v.Args {
- // v uses all registers is reads from, but we only
- // care about marking those containing pointers.
- uevar = addLocs(uevar, arg, true)
- }
- return uevar, kill
-}
-
-type liveRegMask uint32 // only if !go115ReduceLiveness
-
-func (m liveRegMask) niceString(config *ssa.Config) string {
- if m == 0 {
- return "<none>"
- }
- str := ""
- for i, reg := range config.GCRegMap {
- if m&(1<<uint(i)) != 0 {
- if str != "" {
- str += ","
- }
- str += reg.String()
- }
- }
- return str
-}
-
type livenessFuncCache struct {
be []BlockEffects
livenessMap LivenessMap
vars: vars,
idx: idx,
stkptrsize: stkptrsize,
-
- regMapSet: make(map[liveRegMask]int),
}
// Significant sources of allocation are kept in the ssa.Cache
if cap(lc.be) >= f.NumBlocks() {
lv.be = lc.be[:f.NumBlocks()]
}
- lv.livenessMap = LivenessMap{vals: lc.livenessMap.vals, deferreturn: LivenessInvalid}
+ lv.livenessMap = LivenessMap{vals: lc.livenessMap.vals, deferreturn: LivenessDontCare}
lc.livenessMap.vals = nil
}
if lv.be == nil {
for _, b := range f.Blocks {
be := lv.blockEffects(b)
- be.uevar = varRegVec{vars: bulk.next()}
- be.varkill = varRegVec{vars: bulk.next()}
- be.livein = varRegVec{vars: bulk.next()}
- be.liveout = varRegVec{vars: bulk.next()}
+ be.uevar = bulk.next()
+ be.varkill = bulk.next()
+ be.livein = bulk.next()
+ be.liveout = bulk.next()
}
lv.livenessMap.reset()
}
}
-// usedRegs returns the maximum width of the live register map.
-func (lv *Liveness) usedRegs() int32 {
- var any liveRegMask
- for _, live := range lv.regMaps {
- any |= live
- }
- i := int32(0)
- for any != 0 {
- any >>= 1
- i++
- }
- return i
-}
-
// Generates live pointer value maps for arguments and local variables. The
// this argument and the in arguments are always assumed live. The vars
// argument is a slice of *Nodes.
// particular, call Values can have a stack map in case the callee
// grows the stack, but not themselves be a safe-point.
func (lv *Liveness) hasStackMap(v *ssa.Value) bool {
- // The runtime only has safe-points in function prologues, so
- // we only need stack maps at call sites. go:nosplit functions
- // are similar.
- if go115ReduceLiveness || compiling_runtime || lv.f.NoSplit {
- if !v.Op.IsCall() {
- return false
- }
- // typedmemclr and typedmemmove are write barriers and
- // deeply non-preemptible. They are unsafe points and
- // hence should not have liveness maps.
- if sym, ok := v.Aux.(*ssa.AuxCall); ok && (sym.Fn == typedmemclr || sym.Fn == typedmemmove) {
- return false
- }
- return true
+ if !v.Op.IsCall() {
+ return false
}
-
- switch v.Op {
- case ssa.OpInitMem, ssa.OpArg, ssa.OpSP, ssa.OpSB,
- ssa.OpSelect0, ssa.OpSelect1, ssa.OpGetG,
- ssa.OpVarDef, ssa.OpVarLive, ssa.OpKeepAlive,
- ssa.OpPhi:
- // These don't produce code (see genssa).
+ // typedmemclr and typedmemmove are write barriers and
+ // deeply non-preemptible. They are unsafe points and
+ // hence should not have liveness maps.
+ if sym, ok := v.Aux.(*ssa.AuxCall); ok && (sym.Fn == typedmemclr || sym.Fn == typedmemmove) {
return false
}
- return !lv.unsafePoints.Get(int32(v.ID))
+ return true
}
// Initializes the sets for solving the live variables. Visits all the
// effects with the each prog effects.
for j := len(b.Values) - 1; j >= 0; j-- {
pos, e := lv.valueEffects(b.Values[j])
- regUevar, regKill := lv.regEffects(b.Values[j])
if e&varkill != 0 {
- be.varkill.vars.Set(pos)
- be.uevar.vars.Unset(pos)
+ be.varkill.Set(pos)
+ be.uevar.Unset(pos)
}
- be.varkill.regs |= regKill
- be.uevar.regs &^= regKill
if e&uevar != 0 {
- be.uevar.vars.Set(pos)
+ be.uevar.Set(pos)
}
- be.uevar.regs |= regUevar
}
}
}
// These temporary bitvectors exist to avoid successive allocations and
// frees within the loop.
nvars := int32(len(lv.vars))
- newlivein := varRegVec{vars: bvalloc(nvars)}
- newliveout := varRegVec{vars: bvalloc(nvars)}
+ newlivein := bvalloc(nvars)
+ newliveout := bvalloc(nvars)
// Walk blocks in postorder ordering. This improves convergence.
po := lv.f.Postorder()
switch b.Kind {
case ssa.BlockRet:
for _, pos := range lv.cache.retuevar {
- newliveout.vars.Set(pos)
+ newliveout.Set(pos)
}
case ssa.BlockRetJmp:
for _, pos := range lv.cache.tailuevar {
- newliveout.vars.Set(pos)
+ newliveout.Set(pos)
}
case ssa.BlockExit:
// panic exit - nothing to do
// variables at each safe point locations.
func (lv *Liveness) epilogue() {
nvars := int32(len(lv.vars))
- liveout := varRegVec{vars: bvalloc(nvars)}
+ liveout := bvalloc(nvars)
livedefer := bvalloc(nvars) // always-live variables
// If there is a defer (that could recover), then all output
{
// Reserve an entry for function entry.
live := bvalloc(nvars)
- lv.livevars = append(lv.livevars, varRegVec{vars: live})
+ lv.livevars = append(lv.livevars, live)
}
for _, b := range lv.f.Blocks {
be := lv.blockEffects(b)
- firstBitmapIndex := len(lv.livevars)
// Walk forward through the basic block instructions and
// allocate liveness maps for those instructions that need them.
}
live := bvalloc(nvars)
- lv.livevars = append(lv.livevars, varRegVec{vars: live})
+ lv.livevars = append(lv.livevars, live)
}
// walk backward, construct maps at each safe point
live := &lv.livevars[index]
live.Or(*live, liveout)
- live.vars.Or(live.vars, livedefer) // only for non-entry safe points
+ live.Or(*live, livedefer) // only for non-entry safe points
index--
}
// Update liveness information.
pos, e := lv.valueEffects(v)
- regUevar, regKill := lv.regEffects(v)
if e&varkill != 0 {
- liveout.vars.Unset(pos)
+ liveout.Unset(pos)
}
- liveout.regs &^= regKill
if e&uevar != 0 {
- liveout.vars.Set(pos)
+ liveout.Set(pos)
}
- liveout.regs |= regUevar
}
if b == lv.f.Entry {
// Check to make sure only input variables are live.
for i, n := range lv.vars {
- if !liveout.vars.Get(int32(i)) {
+ if !liveout.Get(int32(i)) {
continue
}
if n.Class() == PPARAM {
live.Or(*live, liveout)
}
- // Check that no registers are live across calls.
- // For closure calls, the CALLclosure is the last use
- // of the context register, so it's dead after the call.
- index = int32(firstBitmapIndex)
- for _, v := range b.Values {
- if lv.hasStackMap(v) {
- live := lv.livevars[index]
- if v.Op.IsCall() && live.regs != 0 {
- lv.printDebug()
- v.Fatalf("%v register %s recorded as live at call", lv.fn.Func.Nname, live.regs.niceString(lv.f.Config))
- }
- index++
- }
- }
-
// The liveness maps for this block are now complete. Compact them.
lv.compact(b)
}
// If we have an open-coded deferreturn call, make a liveness map for it.
if lv.fn.Func.OpenCodedDeferDisallowed() {
- lv.livenessMap.deferreturn = LivenessInvalid
+ lv.livenessMap.deferreturn = LivenessDontCare
} else {
lv.livenessMap.deferreturn = LivenessIndex{
stackMapIndex: lv.stackMapSet.add(livedefer),
- regMapIndex: 0, // entry regMap, containing no live registers
isUnsafePoint: false,
}
}
lv.f.Fatalf("%v %L recorded as live on entry", lv.fn.Func.Nname, n)
}
}
- if !go115ReduceLiveness {
- // Check that no registers are live at function entry.
- // The context register, if any, comes from a
- // LoweredGetClosurePtr operation first thing in the function,
- // so it doesn't appear live at entry.
- if regs := lv.regMaps[0]; regs != 0 {
- lv.printDebug()
- lv.f.Fatalf("%v register %s recorded as live on entry", lv.fn.Func.Nname, regs.niceString(lv.f.Config))
- }
- }
}
// Compact coalesces identical bitmaps from lv.livevars into the sets
-// lv.stackMapSet and lv.regMaps.
+// lv.stackMapSet.
//
// Compact clears lv.livevars.
//
// PCDATA tables cost about 100k. So for now we keep using a single index for
// both bitmap lists.
func (lv *Liveness) compact(b *ssa.Block) {
- add := func(live varRegVec, isUnsafePoint bool) LivenessIndex { // only if !go115ReduceLiveness
- // Deduplicate the stack map.
- stackIndex := lv.stackMapSet.add(live.vars)
- // Deduplicate the register map.
- regIndex, ok := lv.regMapSet[live.regs]
- if !ok {
- regIndex = len(lv.regMapSet)
- lv.regMapSet[live.regs] = regIndex
- lv.regMaps = append(lv.regMaps, live.regs)
- }
- return LivenessIndex{stackIndex, regIndex, isUnsafePoint}
- }
pos := 0
if b == lv.f.Entry {
// Handle entry stack map.
- if !go115ReduceLiveness {
- add(lv.livevars[0], false)
- } else {
- lv.stackMapSet.add(lv.livevars[0].vars)
- }
+ lv.stackMapSet.add(lv.livevars[0])
pos++
}
for _, v := range b.Values {
- if go115ReduceLiveness {
- hasStackMap := lv.hasStackMap(v)
- isUnsafePoint := lv.allUnsafe || lv.unsafePoints.Get(int32(v.ID))
- idx := LivenessIndex{StackMapDontCare, StackMapDontCare, isUnsafePoint}
- if hasStackMap {
- idx.stackMapIndex = lv.stackMapSet.add(lv.livevars[pos].vars)
- pos++
- }
- if hasStackMap || isUnsafePoint {
- lv.livenessMap.set(v, idx)
- }
- } else if lv.hasStackMap(v) {
- isUnsafePoint := lv.allUnsafe || lv.unsafePoints.Get(int32(v.ID))
- lv.livenessMap.set(v, add(lv.livevars[pos], isUnsafePoint))
+ hasStackMap := lv.hasStackMap(v)
+ isUnsafePoint := lv.allUnsafe || lv.unsafePoints.Get(int32(v.ID))
+ idx := LivenessIndex{StackMapDontCare, isUnsafePoint}
+ if hasStackMap {
+ idx.stackMapIndex = lv.stackMapSet.add(lv.livevars[pos])
pos++
}
+ if hasStackMap || isUnsafePoint {
+ lv.livenessMap.set(v, idx)
+ }
}
// Reset livevars.
Warnl(pos, s)
}
-func (lv *Liveness) printbvec(printed bool, name string, live varRegVec) bool {
- if live.vars.IsEmpty() && live.regs == 0 {
+func (lv *Liveness) printbvec(printed bool, name string, live bvec) bool {
+ if live.IsEmpty() {
return printed
}
comma := ""
for i, n := range lv.vars {
- if !live.vars.Get(int32(i)) {
+ if !live.Get(int32(i)) {
continue
}
fmt.Printf("%s%s", comma, n.Sym.Name)
comma = ","
}
- fmt.Printf("%s%s", comma, live.regs.niceString(lv.f.Config))
return true
}
-// printeffect is like printbvec, but for valueEffects and regEffects.
-func (lv *Liveness) printeffect(printed bool, name string, pos int32, x bool, regMask liveRegMask) bool {
- if !x && regMask == 0 {
+// printeffect is like printbvec, but for valueEffects.
+func (lv *Liveness) printeffect(printed bool, name string, pos int32, x bool) bool {
+ if !x {
return printed
}
if !printed {
if x {
fmt.Printf("%s", lv.vars[pos].Sym.Name)
}
- for j, reg := range lv.f.Config.GCRegMap {
- if regMask&(1<<uint(j)) != 0 {
- if x {
- fmt.Printf(",")
- }
- x = true
- fmt.Printf("%v", reg)
- }
- }
+
return true
}
pcdata := lv.livenessMap.Get(v)
pos, effect := lv.valueEffects(v)
- regUevar, regKill := lv.regEffects(v)
printed = false
- printed = lv.printeffect(printed, "uevar", pos, effect&uevar != 0, regUevar)
- printed = lv.printeffect(printed, "varkill", pos, effect&varkill != 0, regKill)
+ printed = lv.printeffect(printed, "uevar", pos, effect&uevar != 0)
+ printed = lv.printeffect(printed, "varkill", pos, effect&varkill != 0)
if printed {
fmt.Printf("\n")
}
- if pcdata.StackMapValid() || pcdata.RegMapValid() {
+ if pcdata.StackMapValid() {
fmt.Printf("\tlive=")
printed = false
if pcdata.StackMapValid() {
printed = true
}
}
- if pcdata.RegMapValid() { // only if !go115ReduceLiveness
- regLive := lv.regMaps[pcdata.regMapIndex]
- if regLive != 0 {
- if printed {
- fmt.Printf(",")
- }
- fmt.Printf("%s", regLive.niceString(lv.f.Config))
- printed = true
- }
- }
fmt.Printf("\n")
}
// first word dumped is the total number of bitmaps. The second word is the
// length of the bitmaps. All bitmaps are assumed to be of equal length. The
// remaining bytes are the raw bitmaps.
-func (lv *Liveness) emit() (argsSym, liveSym, regsSym *obj.LSym) {
+func (lv *Liveness) emit() (argsSym, liveSym *obj.LSym) {
// Size args bitmaps to be just large enough to hold the largest pointer.
// First, find the largest Xoffset node we care about.
// (Nodes without pointers aren't in lv.vars; see livenessShouldTrack.)
maxLocals := lv.stkptrsize
// Temporary symbols for encoding bitmaps.
- var argsSymTmp, liveSymTmp, regsSymTmp obj.LSym
+ var argsSymTmp, liveSymTmp obj.LSym
args := bvalloc(int32(maxArgs / int64(Widthptr)))
aoff := duint32(&argsSymTmp, 0, uint32(len(lv.stackMaps))) // number of bitmaps
loff = dbvec(&liveSymTmp, loff, locals)
}
- if !go115ReduceLiveness {
- regs := bvalloc(lv.usedRegs())
- roff := duint32(®sSymTmp, 0, uint32(len(lv.regMaps))) // number of bitmaps
- roff = duint32(®sSymTmp, roff, uint32(regs.n)) // number of bits in each bitmap
- if regs.n > 32 {
- // Our uint32 conversion below won't work.
- Fatalf("GP registers overflow uint32")
- }
-
- if regs.n > 0 {
- for _, live := range lv.regMaps {
- regs.Clear()
- regs.b[0] = uint32(live)
- roff = dbvec(®sSymTmp, roff, regs)
- }
- }
- }
-
// Give these LSyms content-addressable names,
// so that they can be de-duplicated.
// This provides significant binary size savings.
lsym.Set(obj.AttrContentAddressable, true)
})
}
- if !go115ReduceLiveness {
- return makeSym(&argsSymTmp), makeSym(&liveSymTmp), makeSym(®sSymTmp)
- }
- // TODO(go115ReduceLiveness): Remove regsSym result
- return makeSym(&argsSymTmp), makeSym(&liveSymTmp), nil
+ return makeSym(&argsSymTmp), makeSym(&liveSymTmp)
}
// Entry pointer for liveness analysis. Solves for the liveness of
// Emit the live pointer map data structures
ls := e.curfn.Func.lsym
fninfo := ls.Func()
- fninfo.GCArgs, fninfo.GCLocals, fninfo.GCRegs = lv.emit()
+ fninfo.GCArgs, fninfo.GCLocals = lv.emit()
p := pp.Prog(obj.AFUNCDATA)
Addrconst(&p.From, objabi.FUNCDATA_ArgsPointerMaps)
p.To.Name = obj.NAME_EXTERN
p.To.Sym = fninfo.GCLocals
- if !go115ReduceLiveness {
- p = pp.Prog(obj.AFUNCDATA)
- Addrconst(&p.From, objabi.FUNCDATA_RegPointerMaps)
- p.To.Type = obj.TYPE_MEM
- p.To.Name = obj.NAME_EXTERN
- p.To.Sym = fninfo.GCRegs
- }
-
return lv.livenessMap
}