From 5402d40d5b041399392b29e4543f5fc4506197bd Mon Sep 17 00:00:00 2001 From: Jeremy Faller Date: Tue, 18 Aug 2020 16:35:26 -0400 Subject: [PATCH] [dev.link] cmd/link: fix memory growth on dev.link MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit CL 247399 caused memory growth in the linker. Fix this by adjusting how we preallocate the number of symbols we'll need. cmd/compile (Darwin), alloc/op: Loadlib_GC 33.5MB ± 0% 27.3MB ± 0% Change-Id: I34997329ea4412716114df97fc9dad6ad0c171ee Reviewed-on: https://go-review.googlesource.com/c/go/+/249024 Run-TryBot: Jeremy Faller Reviewed-by: Cherry Zhang Reviewed-by: Austin Clements TryBot-Result: Gobot Gobot --- src/cmd/link/internal/ld/lib.go | 2 +- src/cmd/link/internal/loader/loader.go | 36 +++++++++++++++++--------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index caa4566190..a01bdefa37 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -543,7 +543,7 @@ func (ctxt *Link) loadlib() { } // Add non-package symbols and references of externally defined symbols. - ctxt.loader.LoadNonpkgSyms(ctxt.Arch) + ctxt.loader.LoadSyms(ctxt.Arch) // Load symbols from shared libraries, after all Go object symbols are loaded. for _, lib := range ctxt.Library { diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index f149e3c831..ea9cd1bd2e 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -328,7 +328,7 @@ func NewLoader(flags uint32, elfsetstring elfsetstringFunc, reporter *ErrorRepor ldr := &Loader{ start: make(map[*oReader]Sym), objs: []objIdx{{}, {extReader, 0}}, // reserve index 0 for nil symbol, 1 for external symbols - objSyms: make([]objSym, 1, 100000), // reserve index 0 for nil symbol + objSyms: make([]objSym, 1, 1), // This will get overwritten later. extReader: extReader, symsByName: [2]map[string]Sym{make(map[string]Sym, 80000), make(map[string]Sym, 50000)}, // preallocate ~2MB for ABI0 and ~1MB for ABI1 symbols objByPkg: make(map[string]*oReader), @@ -2016,8 +2016,9 @@ func (l *Loader) FuncInfo(i Sym) FuncInfo { return FuncInfo{} } -// Preload a package: add autolibs, add defined package symbols to the symbol table. -// Does not add non-package symbols yet, which will be done in LoadNonpkgSyms. +// Preload a package: adds autolib. +// Does not add defined package or non-packaged symbols to the symbol table. +// These are done in LoadSyms. // Does not read symbol data. // Returns the fingerprint of the object. func (l *Loader) Preload(localSymVersion int, f *bio.Reader, lib *sym.Library, unit *sym.CompilationUnit, length int64) goobj.FingerprintType { @@ -2060,8 +2061,6 @@ func (l *Loader) Preload(localSymVersion int, f *bio.Reader, lib *sym.Library, u } l.addObj(lib.Pkg, or) - st := loadState{l: l} - st.preloadSyms(or, pkgDef) // The caller expects us consuming all the data f.MustSeek(length, os.SEEK_CUR) @@ -2144,17 +2143,30 @@ func (st *loadState) preloadSyms(r *oReader, kind int) { } } -// Add hashed (content-addressable) symbols, non-package symbols, and +// Add syms, hashed (content-addressable) symbols, non-package symbols, and // references to external symbols (which are always named). -func (l *Loader) LoadNonpkgSyms(arch *sys.Arch) { +func (l *Loader) LoadSyms(arch *sys.Arch) { + // Allocate space for symbols, making a guess as to how much space we need. + // This function was determined empirically by looking at the cmd/compile on + // Darwin, and picking factors for hashed and hashed64 syms. + var symSize, hashedSize, hashed64Size int + for _, o := range l.objs[goObjStart:] { + symSize += o.r.ndef + o.r.nhasheddef/2 + o.r.nhashed64def/2 + o.r.NNonpkgdef() + hashedSize += o.r.nhasheddef / 2 + hashed64Size += o.r.nhashed64def / 2 + } + // Index 0 is invalid for symbols. + l.objSyms = make([]objSym, 1, symSize) + l.npkgsyms = l.NSym() - // Preallocate some space (a few hundreds KB) for some symbols. - // As of Go 1.15, linking cmd/compile has ~8000 hashed64 symbols and - // ~13000 hashed symbols. st := loadState{ l: l, - hashed64Syms: make(map[uint64]symAndSize, 10000), - hashedSyms: make(map[goobj.HashType]symAndSize, 15000), + hashed64Syms: make(map[uint64]symAndSize, hashed64Size), + hashedSyms: make(map[goobj.HashType]symAndSize, hashedSize), + } + + for _, o := range l.objs[goObjStart:] { + st.preloadSyms(o.r, pkgDef) } for _, o := range l.objs[goObjStart:] { st.preloadSyms(o.r, hashed64Def) -- 2.48.1