From f204ca305128ad1b500d3ba77aebf5509509a8fc Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 29 Jul 2020 19:44:49 -0400 Subject: [PATCH] [dev.link] cmd/link: drop hash maps after loading MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit The hash maps are used to deduplicate hashed symbols. Once we loaded all the symbols, we no longer need the hash maps. Drop them. Linking cmd/compile, name old live-B new live-B delta Loadlib_GC 13.1M ± 0% 11.3M ± 0% -13.62% (p=0.008 n=5+5) Change-Id: I4bb1f84e1111a56d9e777cd6a68f7d974b60e321 Reviewed-on: https://go-review.googlesource.com/c/go/+/245721 Reviewed-by: Jeremy Faller --- src/cmd/link/internal/loader/loader.go | 54 +++++++++++++-------- src/cmd/link/internal/loader/loader_test.go | 3 +- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 2f4a0efbf4..4da77c6d32 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -204,10 +204,8 @@ type Loader struct { objSyms []objSym // global index mapping to local index - hashed64Syms map[uint64]symAndSize // short hashed (content-addressable) symbols, keyed by content hash - hashedSyms map[goobj2.HashType]symAndSize // hashed (content-addressable) symbols, keyed by content hash - symsByName [2]map[string]Sym // map symbol name to index, two maps are for ABI0 and ABIInternal - extStaticSyms map[nameVer]Sym // externally defined static symbols, keyed by name + symsByName [2]map[string]Sym // map symbol name to index, two maps are for ABI0 and ABIInternal + extStaticSyms map[nameVer]Sym // externally defined static symbols, keyed by name extReader *oReader // a dummy oReader, for external symbols payloadBatch []extSymPayload @@ -285,7 +283,8 @@ type Loader struct { errorReporter *ErrorReporter - npkgsyms int // number of package symbols, for accounting + npkgsyms int // number of package symbols, for accounting + nhashedsyms int // number of hashed symbols, for accounting } const ( @@ -332,8 +331,6 @@ func NewLoader(flags uint32, elfsetstring elfsetstringFunc, reporter *ErrorRepor objs: []objIdx{{}, {extReader, 0}}, // reserve index 0 for nil symbol, 1 for external symbols objSyms: make([]objSym, 1, 100000), // reserve index 0 for nil symbol extReader: extReader, - hashed64Syms: make(map[uint64]symAndSize, 10000), // TODO: adjust preallocation sizes - hashedSyms: make(map[goobj2.HashType]symAndSize, 20000), // TODO: adjust preallocation sizes symsByName: [2]map[string]Sym{make(map[string]Sym, 80000), make(map[string]Sym, 50000)}, // preallocate ~2MB for ABI0 and ~1MB for ABI1 symbols objByPkg: make(map[string]*oReader), outer: make(map[Sym]Sym), @@ -388,7 +385,8 @@ func (l *Loader) addObj(pkg string, r *oReader) Sym { // Add a symbol from an object file, return the global index. // If the symbol already exist, it returns the index of that symbol. -func (l *Loader) addSym(name string, ver int, r *oReader, li uint32, kind int, osym *goobj2.Sym) Sym { +func (st *loadState) addSym(name string, ver int, r *oReader, li uint32, kind int, osym *goobj2.Sym) Sym { + l := st.l if l.extStart != 0 { panic("addSym called after external symbol is created") } @@ -429,17 +427,17 @@ func (l *Loader) addSym(name string, ver int, r *oReader, li uint32, kind int, o if kind == hashed64Def { checkHash = func() (symAndSize, bool) { h64 = r.Hash64(li - uint32(r.ndef)) - s, existed := l.hashed64Syms[h64] + s, existed := st.hashed64Syms[h64] return s, existed } - addToHashMap = func(ss symAndSize) { l.hashed64Syms[h64] = ss } + addToHashMap = func(ss symAndSize) { st.hashed64Syms[h64] = ss } } else { checkHash = func() (symAndSize, bool) { h = r.Hash(li - uint32(r.ndef+r.nhashed64def)) - s, existed := l.hashedSyms[*h] + s, existed := st.hashedSyms[*h] return s, existed } - addToHashMap = func(ss symAndSize) { l.hashedSyms[*h] = ss } + addToHashMap = func(ss symAndSize) { st.hashedSyms[*h] = ss } } siz := osym.Siz() if s, existed := checkHash(); existed { @@ -2070,7 +2068,8 @@ func (l *Loader) Preload(localSymVersion int, f *bio.Reader, lib *sym.Library, u } l.addObj(lib.Pkg, or) - l.preloadSyms(or, pkgDef) + st := loadState{l: l} + st.preloadSyms(or, pkgDef) // The caller expects us consuming all the data f.MustSeek(length, os.SEEK_CUR) @@ -2078,8 +2077,16 @@ func (l *Loader) Preload(localSymVersion int, f *bio.Reader, lib *sym.Library, u return r.Fingerprint() } +// Holds the loader along with temporary states for loading symbols. +type loadState struct { + l *Loader + hashed64Syms map[uint64]symAndSize // short hashed (content-addressable) symbols, keyed by content hash + hashedSyms map[goobj2.HashType]symAndSize // hashed (content-addressable) symbols, keyed by content hash +} + // Preload symbols of given kind from an object. -func (l *Loader) preloadSyms(r *oReader, kind int) { +func (st *loadState) preloadSyms(r *oReader, kind int) { + l := st.l var start, end uint32 switch kind { case pkgDef: @@ -2121,7 +2128,7 @@ func (l *Loader) preloadSyms(r *oReader, kind int) { } v = abiToVer(osym.ABI(), r.version) } - gi := l.addSym(name, v, r, i, kind, osym) + gi := st.addSym(name, v, r, i, kind, osym) r.syms[i] = gi if osym.TopFrame() { l.SetAttrTopFrame(gi, true) @@ -2152,11 +2159,20 @@ func (l *Loader) preloadSyms(r *oReader, kind int) { // references to external symbols (which are always named). func (l *Loader) LoadNonpkgSyms(arch *sys.Arch) { l.npkgsyms = l.NSym() + // Preallocate some space (a few hundreds KB) for some symbols. + // As of Go 1.15, linking cmd/compile has ~8000 hashed64 symbols and + // ~13000 hashed symbols. + st := loadState{ + l: l, + hashed64Syms: make(map[uint64]symAndSize, 10000), + hashedSyms: make(map[goobj2.HashType]symAndSize, 15000), + } for _, o := range l.objs[goObjStart:] { - l.preloadSyms(o.r, hashed64Def) - l.preloadSyms(o.r, hashedDef) - l.preloadSyms(o.r, nonPkgDef) + st.preloadSyms(o.r, hashed64Def) + st.preloadSyms(o.r, hashedDef) + st.preloadSyms(o.r, nonPkgDef) } + l.nhashedsyms = len(st.hashed64Syms) + len(st.hashedSyms) for _, o := range l.objs[goObjStart:] { loadObjRefs(l, o.r, arch) } @@ -2575,7 +2591,7 @@ func (l *Loader) Errorf(s Sym, format string, args ...interface{}) { func (l *Loader) Stat() string { s := fmt.Sprintf("%d symbols, %d reachable\n", l.NSym(), l.NReachableSym()) s += fmt.Sprintf("\t%d package symbols, %d hashed symbols, %d non-package symbols, %d external symbols\n", - l.npkgsyms, len(l.hashed64Syms)+len(l.hashedSyms), int(l.extStart)-l.npkgsyms-len(l.hashed64Syms)-len(l.hashedSyms), l.NSym()-int(l.extStart)) + l.npkgsyms, l.nhashedsyms, int(l.extStart)-l.npkgsyms-l.nhashedsyms, l.NSym()-int(l.extStart)) return s } diff --git a/src/cmd/link/internal/loader/loader_test.go b/src/cmd/link/internal/loader/loader_test.go index 70e0986ac7..af66752f5c 100644 --- a/src/cmd/link/internal/loader/loader_test.go +++ b/src/cmd/link/internal/loader/loader_test.go @@ -21,7 +21,8 @@ import ( // data or relocations). func addDummyObjSym(t *testing.T, ldr *Loader, or *oReader, name string) Sym { idx := uint32(len(ldr.objSyms)) - return ldr.addSym(name, 0, or, idx, nonPkgDef, &goobj2.Sym{}) + st := loadState{l: ldr} + return st.addSym(name, 0, or, idx, nonPkgDef, &goobj2.Sym{}) } func mkLoader() *Loader { -- 2.50.0