From 24950952759cb26144333b0a47eae7da44808eec Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Tue, 24 Sep 2019 17:31:12 -0400 Subject: [PATCH] [dev.link] cmd/link: load full symbol contents after deadcode pass If the new object file format is used, now we load full symbol contents after the deadcode pass, for reachable symbols only. We still load some informations early, like relocations and the contents of type symbols, which are used in the deadcode pass. If we rewrite deadcode to use index directly, we could delay more of the loading (to sym.Symbol), and perhaps delay the creation of sym.Symbol. TODO: internal linking with host objects doesn't work yet. Change-Id: I7d4880e8f150e8709ffac277e62191623440e4cf Reviewed-on: https://go-review.googlesource.com/c/go/+/197258 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/deadcode.go | 10 +- src/cmd/link/internal/ld/go.go | 6 +- src/cmd/link/internal/ld/lib.go | 92 ++++++----- src/cmd/link/internal/ld/link.go | 29 ++++ src/cmd/link/internal/ld/main.go | 3 + src/cmd/link/internal/objfile/objfile2.go | 189 ++++++++++++++++------ 6 files changed, 228 insertions(+), 101 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode.go b/src/cmd/link/internal/ld/deadcode.go index cadb92b43c..575fabc259 100644 --- a/src/cmd/link/internal/ld/deadcode.go +++ b/src/cmd/link/internal/ld/deadcode.go @@ -60,8 +60,8 @@ func deadcode(ctxt *Link) { d.init() d.flood() - callSym := ctxt.Syms.ROLookup("reflect.Value.Call", sym.SymVerABIInternal) - methSym := ctxt.Syms.ROLookup("reflect.Value.Method", sym.SymVerABIInternal) + callSym := ctxt.Lookup("reflect.Value.Call", sym.SymVerABIInternal) + methSym := ctxt.Lookup("reflect.Value.Method", sym.SymVerABIInternal) reflectSeen := false if ctxt.DynlinkingGo() { @@ -283,7 +283,7 @@ func (d *deadcodepass) init() { // We don't keep the go.plugin.exports symbol, // but we do keep the symbols it refers to. - exports := d.ctxt.Syms.ROLookup("go.plugin.exports", 0) + exports := d.ctxt.Lookup("go.plugin.exports", 0) if exports != nil { for i := range exports.R { d.mark(exports.R[i].Sym, nil) @@ -298,9 +298,9 @@ func (d *deadcodepass) init() { for _, name := range names { // Mark symbol as an data/ABI0 symbol. - d.mark(d.ctxt.Syms.ROLookup(name, 0), nil) + d.mark(d.ctxt.Lookup(name, 0), nil) // Also mark any Go functions (internal ABI). - d.mark(d.ctxt.Syms.ROLookup(name, sym.SymVerABIInternal), nil) + d.mark(d.ctxt.Lookup(name, sym.SymVerABIInternal), nil) } } diff --git a/src/cmd/link/internal/ld/go.go b/src/cmd/link/internal/ld/go.go index d1d68b0704..13fbbed10f 100644 --- a/src/cmd/link/internal/ld/go.go +++ b/src/cmd/link/internal/ld/go.go @@ -169,7 +169,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) { if i := strings.Index(remote, "#"); i >= 0 { remote, q = remote[:i], remote[i+1:] } - s := ctxt.Syms.Lookup(local, 0) + s := ctxt.LookupOrCreate(local, 0) if s.Type == 0 || s.Type == sym.SXREF || s.Type == sym.SHOSTOBJ { s.SetDynimplib(lib) s.SetExtname(remote) @@ -188,7 +188,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) { } local := f[1] - s := ctxt.Syms.Lookup(local, 0) + s := ctxt.LookupOrCreate(local, 0) s.Type = sym.SHOSTOBJ s.Size = 0 continue @@ -209,7 +209,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) { // functions. Link.loadlib will resolve any // ABI aliases we find here (since we may not // yet know it's an alias). - s := ctxt.Syms.Lookup(local, 0) + s := ctxt.LookupOrCreate(local, 0) switch ctxt.BuildMode { case BuildModeCShared, BuildModeCArchive, BuildModePlugin: diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index f11adbcfb6..2ebd5d333c 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -401,21 +401,24 @@ func (ctxt *Link) loadlib() { } } - // XXX do it here for now if *flagNewobj { - ctxt.loadlibfull() - } - - for _, lib := range ctxt.Library { - if lib.Shlib != "" { - if ctxt.Debugvlog > 1 { - ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), lib.Shlib, lib.Objref) + // Add references of externally defined symbols. + for _, lib := range ctxt.Library { + for _, r := range lib.Readers { + objfile.LoadRefs(ctxt.loader, r.Reader, lib, ctxt.Arch, ctxt.Syms, r.Version) } - ldshlibsyms(ctxt, lib.Shlib) + } + + // Load cgo directives. + for _, p := range ctxt.cgodata { + loadcgo(ctxt, p[0], p[1], p[2]) } } - iscgo = ctxt.Syms.ROLookup("x_cgo_init", 0) != nil + iscgo = ctxt.Lookup("x_cgo_init", 0) != nil + + // Record whether we can use plugins. + ctxt.canUsePlugins = (ctxt.Lookup("plugin.Open", sym.SymVerABIInternal) != nil) // We now have enough information to determine the link mode. determineLinkMode(ctxt) @@ -448,8 +451,34 @@ func (ctxt *Link) loadlib() { dynexp[i] = t } + for _, lib := range ctxt.Library { + if lib.Shlib != "" { + if ctxt.Debugvlog > 1 { + ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), lib.Shlib, lib.Objref) + } + ldshlibsyms(ctxt, lib.Shlib) + } + } + + if ctxt.LinkMode == LinkExternal && !iscgo && ctxt.LibraryByPkg["runtime/cgo"] == nil && !(objabi.GOOS == "darwin" && (ctxt.Arch.Family == sys.AMD64 || ctxt.Arch.Family == sys.I386)) { + // This indicates a user requested -linkmode=external. + // The startup code uses an import of runtime/cgo to decide + // whether to initialize the TLS. So give it one. This could + // be handled differently but it's an unusual case. + if lib := loadinternal(ctxt, "runtime/cgo"); lib != nil { + if lib.Shlib != "" { + ldshlibsyms(ctxt, lib.Shlib) + } else { + if ctxt.BuildMode == BuildModeShared || ctxt.linkShared { + Exitf("cannot implicitly include runtime/cgo in a shared library") + } + loadobjfile(ctxt, lib) + } + } + } + // In internal link mode, read the host object files. - if ctxt.LinkMode == LinkInternal { + if ctxt.LinkMode == LinkInternal && len(hostobj) != 0 { // Drop all the cgo_import_static declarations. // Turns out we won't be needing them. for _, s := range ctxt.Syms.Allsym { @@ -510,34 +539,23 @@ func (ctxt *Link) loadlib() { */ } } - } else { + } else if ctxt.LinkMode == LinkExternal { hostlinksetup(ctxt) } // We've loaded all the code now. ctxt.Loaded = true - // Record whether we can use plugins. - ctxt.canUsePlugins = (ctxt.Syms.ROLookup("plugin.Open", sym.SymVerABIInternal) != nil) + importcycles() - if ctxt.LinkMode == LinkExternal && !iscgo && ctxt.LibraryByPkg["runtime/cgo"] == nil && !(objabi.GOOS == "darwin" && (ctxt.Arch.Family == sys.AMD64 || ctxt.Arch.Family == sys.I386)) { - // This indicates a user requested -linkmode=external. - // The startup code uses an import of runtime/cgo to decide - // whether to initialize the TLS. So give it one. This could - // be handled differently but it's an unusual case. - if lib := loadinternal(ctxt, "runtime/cgo"); lib != nil { - if lib.Shlib != "" { - ldshlibsyms(ctxt, lib.Shlib) - } else { - if ctxt.BuildMode == BuildModeShared || ctxt.linkShared { - Exitf("cannot implicitly include runtime/cgo in a shared library") - } - loadobjfile(ctxt, lib) + // For now, load relocations for dead-code elimination. + if *flagNewobj { + for _, lib := range ctxt.Library { + for _, r := range lib.Readers { + objfile.LoadReloc(ctxt.loader, r.Reader, lib, r.Version, ctxt.LibraryByPkg) } } } - - importcycles() } // Set up flags and special symbols depending on the platform build mode. @@ -1907,7 +1925,7 @@ func ldshlibsyms(ctxt *Link, shlib string) { ver = sym.SymVerABIInternal } - lsym := ctxt.Syms.Lookup(elfsym.Name, ver) + lsym := ctxt.LookupOrCreate(elfsym.Name, ver) // Because loadlib above loads all .a files before loading any shared // libraries, any non-dynimport symbols we find that duplicate symbols // already loaded should be ignored (the symbols from the .a files @@ -2526,17 +2544,10 @@ func dfs(lib *sym.Library, mark map[*sym.Library]markKind, order *[]*sym.Library } func (ctxt *Link) loadlibfull() { - // Add references of externally defined symbols. - for _, lib := range ctxt.Library { - for _, r := range lib.Readers { - objfile.LoadRefs(ctxt.loader, r.Reader, lib, ctxt.Arch, ctxt.Syms, r.Version) - } - } - // Load full symbol contents, resolve indexed references. for _, lib := range ctxt.Library { for _, r := range lib.Readers { - objfile.LoadFull(ctxt.loader, r.Reader, lib, ctxt.Syms, r.Version, ctxt.LibraryByPkg) + objfile.LoadFull(ctxt.loader, r.Reader, lib, r.Version, ctxt.LibraryByPkg) } } @@ -2546,11 +2557,6 @@ func (ctxt *Link) loadlibfull() { ctxt.Syms.Add(s) } } - - // Now load cgo directives. - for _, p := range ctxt.cgodata { - loadcgo(ctxt, p[0], p[1], p[2]) - } } func (ctxt *Link) dumpsyms() { diff --git a/src/cmd/link/internal/ld/link.go b/src/cmd/link/internal/ld/link.go index bbdb0e50ed..dfb686f038 100644 --- a/src/cmd/link/internal/ld/link.go +++ b/src/cmd/link/internal/ld/link.go @@ -176,3 +176,32 @@ func addImports(ctxt *Link, l *sym.Library, pn string) { } l.ImportStrings = nil } + +// convenient helper during the transition period. +func (ctxt *Link) Lookup(name string, ver int) *sym.Symbol { + if *flagNewobj { + i := ctxt.loader.Lookup(name, ver) + if i == 0 { + return nil + } + return ctxt.loader.Syms[i] + } else { + return ctxt.Syms.ROLookup(name, ver) + } +} + +// convenient helper during the transition period. +func (ctxt *Link) LookupOrCreate(name string, ver int) *sym.Symbol { + if *flagNewobj { + i := ctxt.loader.Lookup(name, ver) + if i != 0 { + return ctxt.loader.Syms[i] + } + ctxt.loader.AddExtSym(name, ver) + s := ctxt.Syms.Newsym(name, ver) + ctxt.loader.Syms = append(ctxt.loader.Syms, s) + return s + } else { + return ctxt.Syms.Lookup(name, ver) + } +} diff --git a/src/cmd/link/internal/ld/main.go b/src/cmd/link/internal/ld/main.go index 759e7fb744..e667afecc1 100644 --- a/src/cmd/link/internal/ld/main.go +++ b/src/cmd/link/internal/ld/main.go @@ -209,6 +209,9 @@ func Main(arch *sys.Arch, theArch Arch) { ctxt.loadlib() deadcode(ctxt) + if *flagNewobj { + ctxt.loadlibfull() // XXX do it here for now + } ctxt.linksetup() ctxt.dostrdata() diff --git a/src/cmd/link/internal/objfile/objfile2.go b/src/cmd/link/internal/objfile/objfile2.go index 1908d21c93..a5bd91d3ab 100644 --- a/src/cmd/link/internal/objfile/objfile2.go +++ b/src/cmd/link/internal/objfile/objfile2.go @@ -225,7 +225,10 @@ func preprocess(arch *sys.Arch, s *sym.Symbol) { } } -func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, localSymVersion int, libByPkg map[string]*sym.Library) { +// Load relocations for building the dependency graph in deadcode pass. +// For now, we load symbol types, relocations, gotype, and the contents +// of type symbols, which are needed in deadcode. +func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int, libByPkg map[string]*sym.Library) { // PkgIdx pkglist := r.Pkglist() @@ -262,7 +265,6 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, return l.Syms[i] } - pcdataBase := r.PcdataBase() for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { s := l.Syms[istart+i] if s == nil || s.Name == "" { @@ -272,28 +274,30 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, osym := goobj2.Sym{} osym.Read(r, r.SymOff(i)) name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) - if s.Name != name { + if s.Name != name { // Sanity check. We can remove it in the final version. fmt.Println("name mismatch:", lib, i, s.Name, name) panic("name mismatch") } - dupok := osym.Flag&goobj2.SymFlagDupok != 0 - local := osym.Flag&goobj2.SymFlagLocal != 0 - makeTypelink := osym.Flag&goobj2.SymFlagTypelink != 0 - nreloc := r.NReloc(i) - datasize := r.DataSize(i) - size := osym.Siz - - t := sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)] if s.Type != 0 && s.Type != sym.SXREF { fmt.Println("symbol already processed:", lib, i, s) panic("symbol already processed") } - // Symbol data - s.P = r.BytesAt(r.DataOff(i), datasize) + t := sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)] + if t == sym.SXREF { + log.Fatalf("bad sxref") + } + if t == 0 { + log.Fatalf("missing type for %s in %s", s.Name, lib) + } + if t == sym.SBSS && (s.Type == sym.SRODATA || s.Type == sym.SNOPTRBSS) { + t = s.Type + } + s.Type = t // Reloc + nreloc := r.NReloc(i) s.R = make([]sym.Reloc, nreloc) for j := range s.R { rel := goobj2.Reloc{} @@ -307,9 +311,13 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, } } - // Aux symbol info - isym := -1 - funcdata := make([]goobj2.SymRef, 0, 4) + // XXX deadcode needs symbol data for type symbols. Read it now. + if strings.HasPrefix(name, "type.") { + s.P = r.BytesAt(r.DataOff(i), r.DataSize(i)) + s.Size = int64(osym.Siz) + } + + // Aux symbol naux := r.NAux(i) for j := 0; j < naux; j++ { a := goobj2.Aux{} @@ -320,13 +328,115 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, if typ != nil { s.Gotype = typ } + case goobj2.AuxFuncdata: + pc := s.FuncInfo + if pc == nil { + pc = &sym.FuncInfo{Funcdata: make([]*sym.Symbol, 0, 4)} + s.FuncInfo = pc + } + pc.Funcdata = append(pc.Funcdata, resolveSymRef(a.Sym)) + } + } + + if s.Type == sym.STEXT { + dupok := osym.Flag&goobj2.SymFlagDupok != 0 + if !dupok { + if s.Attr.OnList() { + log.Fatalf("symbol %s listed multiple times", s.Name) + } + s.Attr |= sym.AttrOnList + lib.Textp = append(lib.Textp, s) + } else { + // there may ba a dup in another package + // put into a temp list and add to text later + lib.DupTextSyms = append(lib.DupTextSyms, s) + } + } + } +} + +// Load full contents. +// TODO: For now, some contents are already load in LoadReloc. Maybe +// we should combine LoadReloc back into this, once we rewrite deadcode +// pass to use index directly. +func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int, libByPkg map[string]*sym.Library) { + // PkgIdx + pkglist := r.Pkglist() + + pkgprefix := objabi.PathToPrefix(lib.Pkg) + "." + istart := l.StartIndex(r) + + resolveSymRef := func(s goobj2.SymRef) *sym.Symbol { + var rr *goobj2.Reader + switch p := s.PkgIdx; p { + case goobj2.PkgIdxInvalid: + if s.SymIdx != 0 { + panic("bad sym ref") + } + return nil + case goobj2.PkgIdxNone: + // Resolve by name + i := int(s.SymIdx) + r.NSym() + osym := goobj2.Sym{} + osym.Read(r, r.SymOff(i)) + name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) + v := abiToVer(osym.ABI, localSymVersion) + nv := nameVer{name, v} + i = l.symsByName[nv] + return l.Syms[i] + case goobj2.PkgIdxSelf: + rr = r + default: + pkg := pkglist[p] + rr = libByPkg[pkg].Readers[0].Reader // typically Readers[0] is go object (others are asm) + } + i := l.ToGlobal(rr, int(s.SymIdx)) + return l.Syms[i] + } + + pcdataBase := r.PcdataBase() + for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ { + s := l.Syms[istart+i] + if s == nil || s.Name == "" { + continue + } + if !s.Attr.Reachable() && (s.Type < sym.SDWARFSECT || s.Type > sym.SDWARFLINES) { + // No need to load unreachable symbols. + // XXX DWARF symbols may be used but are not marked reachable. + continue + } + + osym := goobj2.Sym{} + osym.Read(r, r.SymOff(i)) + name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1) + if s.Name != name { // Sanity check. We can remove it in the final version. + fmt.Println("name mismatch:", lib, i, s.Name, name) + panic("name mismatch") + } + + dupok := osym.Flag&goobj2.SymFlagDupok != 0 + local := osym.Flag&goobj2.SymFlagLocal != 0 + makeTypelink := osym.Flag&goobj2.SymFlagTypelink != 0 + datasize := r.DataSize(i) + size := osym.Siz + + // Symbol data + s.P = r.BytesAt(r.DataOff(i), datasize) + + // Aux symbol info + isym := -1 + naux := r.NAux(i) + for j := 0; j < naux; j++ { + a := goobj2.Aux{} + a.Read(r, r.AuxOff(i, j)) + switch a.Type { + case goobj2.AuxGotype, goobj2.AuxFuncdata: + // already loaded case goobj2.AuxFuncInfo: if a.Sym.PkgIdx != goobj2.PkgIdxSelf { panic("funcinfo symbol not defined in current package") } isym = int(a.Sym.SymIdx) - case goobj2.AuxFuncdata: - funcdata = append(funcdata, a.Sym) default: panic("unknown aux type") } @@ -336,16 +446,6 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, if dupok { s.Attr |= sym.AttrDuplicateOK } - if t == sym.SXREF { - log.Fatalf("bad sxref") - } - if t == 0 { - log.Fatalf("missing type for %s in %s", s.Name, lib) - } - if t == sym.SBSS && (s.Type == sym.SRODATA || s.Type == sym.SNOPTRBSS) { - t = s.Type - } - s.Type = t if s.Size < int64(size) { s.Size = int64(size) } @@ -355,17 +455,6 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, if s.Type != sym.STEXT { continue } - if !dupok { - if s.Attr.OnList() { - log.Fatalf("symbol %s listed multiple times", s.Name) - } - s.Attr |= sym.AttrOnList - lib.Textp = append(lib.Textp, s) - } else { - // there may ba a dup in another package - // put into a temp list and add to text later - lib.DupTextSyms = append(lib.DupTextSyms, s) - } // FuncInfo if isym == -1 { @@ -389,15 +478,16 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, } info.Pcdata = append(info.Pcdata, info.PcdataEnd) // for the ease of knowing where it ends - pc := &sym.FuncInfo{ - Args: int32(info.Args), - Locals: int32(info.Locals), - Pcdata: make([]sym.Pcdata, len(info.Pcdata)-1), // -1 as we appended one above - Funcdata: make([]*sym.Symbol, len(info.Funcdataoff)), - Funcdataoff: make([]int64, len(info.Funcdataoff)), - File: make([]*sym.Symbol, len(info.File)), - } - s.FuncInfo = pc + pc := s.FuncInfo + if pc == nil { + pc = &sym.FuncInfo{} + s.FuncInfo = pc + } + pc.Args = int32(info.Args) + pc.Locals = int32(info.Locals) + pc.Pcdata = make([]sym.Pcdata, len(info.Pcdata)-1) // -1 as we appended one above + pc.Funcdataoff = make([]int64, len(info.Funcdataoff)) + pc.File = make([]*sym.Symbol, len(info.File)) pc.Pcsp.P = r.BytesAt(pcdataBase+info.Pcsp, int(info.Pcfile-info.Pcsp)) pc.Pcfile.P = r.BytesAt(pcdataBase+info.Pcfile, int(info.Pcline-info.Pcfile)) pc.Pcline.P = r.BytesAt(pcdataBase+info.Pcline, int(info.Pcinline-info.Pcline)) @@ -405,8 +495,7 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, for k := range pc.Pcdata { pc.Pcdata[k].P = r.BytesAt(pcdataBase+info.Pcdata[k], int(info.Pcdata[k+1]-info.Pcdata[k])) } - for k := range pc.Funcdata { - pc.Funcdata[k] = resolveSymRef(funcdata[k]) + for k := range pc.Funcdataoff { pc.Funcdataoff[k] = int64(info.Funcdataoff[k]) } for k := range pc.File { -- 2.48.1