[dev.link] cmd/link: emit ELF relocations in mmap

Currently, ELF relocations are generated sequentially in the heap
and flushed to output file periodically. In fact, in some cases,
the output size of the relocation records can be easily computed,
as a relocation entry has fixed size. We only need to count the
number of relocation records to compute the size.

Once the size is computed, we can mmap the output with the proper
size, and directly write relocation records in the mapped memory.
It also opens the possibility of writing relocations in parallel
(not done in this CL).

Note: on some architectures, a Go relocation may turn into
multiple ELF relocations, which makes size calculation harder.
This CL does not handle those cases, and it still writes
sequentially in the heap there.

Linking cmd/compile with external linking,

name          old time/op    new time/op    delta
Asmb2            190ms ± 2%     141ms ± 4%  -25.74%  (p=0.000 n=10+10)

name          old alloc/op   new alloc/op   delta
Asmb2_GC        66.8MB ± 0%     8.2MB ± 0%  -87.79%  (p=0.008 n=5+5)

name          old live-B     new live-B     delta
Asmb2_GC         66.9M ± 0%     55.2M ± 0%  -17.58%  (p=0.008 n=5+5)

Change-Id: If7056bbe909dc90033eef6b9c4891fcca310602c
Reviewed-on: https://go-review.googlesource.com/c/go/+/240399
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Than McIntosh <thanm@google.com>

diff --git a/src/cmd/link/internal/amd64/obj.go b/src/cmd/link/internal/amd64/obj.go
index 4c525743feeafcd2bda6619a88acbd799fb9656a..fcc2499cb0f3ed4488a589102c993814f6ed9a21 100644 (file)
--- a/src/cmd/link/internal/amd64/obj.go
+++ b/src/cmd/link/internal/amd64/obj.go
@@ -61,6 +61,7 @@ func Init() (*sys.Arch, ld.Arch) {
                Archreloc:        archreloc,
                Archrelocvariant: archrelocvariant,
                Elfreloc1:        elfreloc1,
+               ElfrelocSize:     24,
                Elfsetupplt:      elfsetupplt,
                Gentext:          gentext,
                Machoreloc1:      machoreloc1,

diff --git a/src/cmd/link/internal/arm/obj.go b/src/cmd/link/internal/arm/obj.go
index 1a572985b6620aa1fa4bec5216bf0eb714822eae..f25f735b0bbd2fcc77b309476085641d8d810c6e 100644 (file)
--- a/src/cmd/link/internal/arm/obj.go
+++ b/src/cmd/link/internal/arm/obj.go
@@ -54,6 +54,7 @@ func Init() (*sys.Arch, ld.Arch) {
                Archrelocvariant: archrelocvariant,
                Trampoline:       trampoline,
                Elfreloc1:        elfreloc1,
+               ElfrelocSize:     8,
                Elfsetupplt:      elfsetupplt,
                Gentext:          gentext,
                Machoreloc1:      machoreloc1,

diff --git a/src/cmd/link/internal/ld/data.go b/src/cmd/link/internal/ld/data.go
index be1af4bcae37b8518ccdbe03dd8b9aebb2270b57..7dc2c4662dde28fb8338c36a5b1350d7396ba646 100644 (file)
--- a/src/cmd/link/internal/ld/data.go
+++ b/src/cmd/link/internal/ld/data.go
@@ -47,6 +47,7 @@ import (
        "strconv"
        "strings"
        "sync"
+       "sync/atomic"
 )
 
 // isRuntimeDepPkg reports whether pkg is the runtime package or its dependency
@@ -581,6 +582,7 @@ func (st *relocSymState) relocsym(s loader.Sym, P []byte) {
        if len(extRelocs) != 0 {
                st.finalizeExtRelocSlice(extRelocs)
                ldr.SetExtRelocs(s, extRelocs)
+               atomic.AddUint32(&ldr.SymSect(s).Relcount, uint32(len(extRelocs)))
        }
 }
 

diff --git a/src/cmd/link/internal/ld/elf.go b/src/cmd/link/internal/ld/elf.go
index 80612c4684d331d53e420b35a770d6c9d70c7e67..bb4e1715f471f6bfd37af86c983fdfa76cb400c6 100644 (file)
--- a/src/cmd/link/internal/ld/elf.go
+++ b/src/cmd/link/internal/ld/elf.go
@@ -1397,11 +1397,29 @@ func elfrelocsect(ctxt *Link, sect *sym.Section, syms []loader.Sym) {
 }
 
 func elfEmitReloc(ctxt *Link) {
-
        for ctxt.Out.Offset()&7 != 0 {
                ctxt.Out.Write8(0)
        }
 
+       // Precompute the size needed for the reloc records if we can
+       // Mmap the output buffer with the proper size.
+       //
+       // TODO: on some architectures, one Go relocation may turn to
+       // multiple ELF relocations, which makes the size not fixed.
+       // Handle this case better. Maybe increment the counter by the
+       // number of external reloc records in relocsym.
+       var sz, filesz int64
+       if thearch.ElfrelocSize != 0 {
+               for _, seg := range Segments {
+                       for _, sect := range seg.Sections {
+                               sz += int64(thearch.ElfrelocSize * sect.Relcount)
+                       }
+               }
+               filesz = ctxt.Out.Offset() + sz
+               ctxt.Out.Mmap(uint64(filesz))
+       }
+
+       // Now emits the records.
        for _, sect := range Segtext.Sections {
                if sect.Name == ".text" {
                        elfrelocsect(ctxt, sect, ctxt.Textp)
@@ -1428,6 +1446,11 @@ func elfEmitReloc(ctxt *Link) {
                }
                elfrelocsect(ctxt, sect, si.syms)
        }
+
+       // sanity check
+       if thearch.ElfrelocSize != 0 && ctxt.Out.Offset() != filesz {
+               panic("elfEmitReloc: size mismatch")
+       }
 }
 
 func addgonote(ctxt *Link, sectionName string, tag uint32, desc []byte) {

diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go
index fbf72f671510782bd83f8da48ce39cfdccfdfe6f..d160139fd57b3d167d0b88615d8a55764a56baea 100644 (file)
--- a/src/cmd/link/internal/ld/lib.go
+++ b/src/cmd/link/internal/ld/lib.go
@@ -236,12 +236,13 @@ type Arch struct {
        Asmb  func(*Link, *loader.Loader)
        Asmb2 func(*Link, *loader.Loader)
 
-       Elfreloc1   func(*Link, *loader.Loader, loader.Sym, loader.ExtRelocView, int64) bool
-       Elfsetupplt func(ctxt *Link, plt, gotplt *loader.SymbolBuilder, dynamic loader.Sym)
-       Gentext     func(*Link, *loader.Loader)
-       Machoreloc1 func(*sys.Arch, *OutBuf, *loader.Loader, loader.Sym, loader.ExtRelocView, int64) bool
-       PEreloc1    func(*sys.Arch, *OutBuf, *loader.Loader, loader.Sym, loader.ExtRelocView, int64) bool
-       Xcoffreloc1 func(*sys.Arch, *OutBuf, *loader.Loader, loader.Sym, loader.ExtRelocView, int64) bool
+       Elfreloc1    func(*Link, *loader.Loader, loader.Sym, loader.ExtRelocView, int64) bool
+       ElfrelocSize uint32 // size of an ELF relocation record, must match Elfreloc1. Currently this can be 0, meaning that the size is not fixed (a Go reloc may turn into multiple ELF reloc).
+       Elfsetupplt  func(ctxt *Link, plt, gotplt *loader.SymbolBuilder, dynamic loader.Sym)
+       Gentext      func(*Link, *loader.Loader)
+       Machoreloc1  func(*sys.Arch, *OutBuf, *loader.Loader, loader.Sym, loader.ExtRelocView, int64) bool
+       PEreloc1     func(*sys.Arch, *OutBuf, *loader.Loader, loader.Sym, loader.ExtRelocView, int64) bool
+       Xcoffreloc1  func(*sys.Arch, *OutBuf, *loader.Loader, loader.Sym, loader.ExtRelocView, int64) bool
 
        // TLSIEtoLE converts a TLS Initial Executable relocation to
        // a TLS Local Executable relocation.
@@ -310,6 +311,8 @@ var (
        Segrelrodata sym.Segment
        Segdata      sym.Segment
        Segdwarf     sym.Segment
+
+       Segments = []*sym.Segment{&Segtext, &Segrodata, &Segrelrodata, &Segdata, &Segdwarf}
 )
 
 const pkgdef = "__.PKGDEF"

diff --git a/src/cmd/link/internal/ld/outbuf.go b/src/cmd/link/internal/ld/outbuf.go
index b474067dd9103ecf786b1dc774e4e1666eaa6774..f0178288a6b01d2e99a1d476769ec1eedf21791a 100644 (file)
--- a/src/cmd/link/internal/ld/outbuf.go
+++ b/src/cmd/link/internal/ld/outbuf.go
@@ -149,13 +149,10 @@ func (out *OutBuf) copyHeap() bool {
        bufLen := len(out.buf)
        heapLen := len(out.heap)
        total := uint64(bufLen + heapLen)
-       out.munmap()
        if heapLen != 0 {
-               if err := out.Mmap(total); err != nil {
+               if err := out.Mmap(total); err != nil { // Mmap will copy out.heap over to out.buf
                        panic(err)
                }
-               copy(out.buf[bufLen:], out.heap[:heapLen])
-               out.heap = out.heap[:0]
        }
        return true
 }

diff --git a/src/cmd/link/internal/ld/outbuf_mmap.go b/src/cmd/link/internal/ld/outbuf_mmap.go
index 7280027e92c708c76e55ca44f40b06c52abf1cff..53b14b09cc9b8d75783f12495ad277eeaaf50094 100644 (file)
--- a/src/cmd/link/internal/ld/outbuf_mmap.go
+++ b/src/cmd/link/internal/ld/outbuf_mmap.go
@@ -10,7 +10,15 @@ import (
        "syscall"
 )
 
+// Mmap maps the output file with the given size. It unmaps the old mapping
+// if it is already mapped. It also flushes any in-heap data to the new
+// mapping.
 func (out *OutBuf) Mmap(filesize uint64) (err error) {
+       oldlen := len(out.buf)
+       if oldlen != 0 {
+               out.munmap()
+       }
+
        for {
                if err = out.fallocate(filesize); err != syscall.EINTR {
                        break
@@ -29,7 +37,17 @@ func (out *OutBuf) Mmap(filesize uint64) (err error) {
                Exitf("resize output file failed: %v", err)
        }
        out.buf, err = syscall.Mmap(int(out.f.Fd()), 0, int(filesize), syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED|syscall.MAP_FILE)
-       return err
+       if err != nil {
+               return err
+       }
+
+       // copy heap to new mapping
+       if uint64(oldlen+len(out.heap)) > filesize {
+               panic("mmap size too small")
+       }
+       copy(out.buf[oldlen:], out.heap)
+       out.heap = out.heap[:0]
+       return nil
 }
 
 func (out *OutBuf) munmap() {

diff --git a/src/cmd/link/internal/ld/outbuf_nommap.go b/src/cmd/link/internal/ld/outbuf_nommap.go
index bad01dc6d56f0f4ee59849e990f597046c9eaa0e..6b4025384b42788e7405fc9036c74ebea2447c9a 100644 (file)
--- a/src/cmd/link/internal/ld/outbuf_nommap.go
+++ b/src/cmd/link/internal/ld/outbuf_nommap.go
@@ -6,9 +6,16 @@
 
 package ld
 
+// Mmap allocates an in-heap output buffer with the given size. It copies
+// any old data (if any) to the new buffer.
 func (out *OutBuf) Mmap(filesize uint64) error {
        // We need space to put all the symbols before we apply relocations.
+       oldheap := out.heap
+       if filesize < uint64(len(oldheap)) {
+               panic("mmap size too small")
+       }
        out.heap = make([]byte, filesize)
+       copy(out.heap, oldheap)
        return nil
 }
 

diff --git a/src/cmd/link/internal/ld/outbuf_windows.go b/src/cmd/link/internal/ld/outbuf_windows.go
index 807c0e227d2a6d5e5d3369f35c2387d4b91f205c..60dc1ab92dd0bd630e24dcfa93f228f0310104d2 100644 (file)
--- a/src/cmd/link/internal/ld/outbuf_windows.go
+++ b/src/cmd/link/internal/ld/outbuf_windows.go
@@ -10,7 +10,15 @@ import (
        "unsafe"
 )
 
+// Mmap maps the output file with the given size. It unmaps the old mapping
+// if it is already mapped. It also flushes any in-heap data to the new
+// mapping.
 func (out *OutBuf) Mmap(filesize uint64) error {
+       oldlen := len(out.buf)
+       if oldlen != 0 {
+               out.munmap()
+       }
+
        err := out.f.Truncate(int64(filesize))
        if err != nil {
                Exitf("resize output file failed: %v", err)
@@ -28,6 +36,13 @@ func (out *OutBuf) Mmap(filesize uint64) error {
                return err
        }
        *(*reflect.SliceHeader)(unsafe.Pointer(&out.buf)) = reflect.SliceHeader{Data: ptr, Len: int(filesize), Cap: int(filesize)}
+
+       // copy heap to new mapping
+       if uint64(oldlen+len(out.heap)) > filesize {
+               panic("mmap size too small")
+       }
+       copy(out.buf[oldlen:], out.heap)
+       out.heap = out.heap[:0]
        return nil
 }
 

diff --git a/src/cmd/link/internal/mips/obj.go b/src/cmd/link/internal/mips/obj.go
index cc3cc431b782f39bb194cb1b6dff8f31706cf6da..e59c382bfa49f848c3ee3527c75a50a844e6588b 100644 (file)
--- a/src/cmd/link/internal/mips/obj.go
+++ b/src/cmd/link/internal/mips/obj.go
@@ -53,6 +53,7 @@ func Init() (*sys.Arch, ld.Arch) {
                Archreloc:        archreloc,
                Archrelocvariant: archrelocvariant,
                Elfreloc1:        elfreloc1,
+               ElfrelocSize:     8,
                Elfsetupplt:      elfsetupplt,
                Gentext:          gentext,
                Machoreloc1:      machoreloc1,

diff --git a/src/cmd/link/internal/mips64/obj.go b/src/cmd/link/internal/mips64/obj.go
index 449a2009288a91f8649ef2e9ae379ac4fbf88064..6ef27cedb934f22c911ad9c5f809048a3322fc81 100644 (file)
--- a/src/cmd/link/internal/mips64/obj.go
+++ b/src/cmd/link/internal/mips64/obj.go
@@ -52,6 +52,7 @@ func Init() (*sys.Arch, ld.Arch) {
                Archreloc:        archreloc,
                Archrelocvariant: archrelocvariant,
                Elfreloc1:        elfreloc1,
+               ElfrelocSize:     24,
                Elfsetupplt:      elfsetupplt,
                Gentext:          gentext,
                Machoreloc1:      machoreloc1,

diff --git a/src/cmd/link/internal/s390x/obj.go b/src/cmd/link/internal/s390x/obj.go
index bb62fe179ff9c5cdd3482f4776f07f2a468097cd..8acc1d491760890cc4ec9b9c39f0ee06645f8487 100644 (file)
--- a/src/cmd/link/internal/s390x/obj.go
+++ b/src/cmd/link/internal/s390x/obj.go
@@ -51,6 +51,7 @@ func Init() (*sys.Arch, ld.Arch) {
                Archreloc:        archreloc,
                Archrelocvariant: archrelocvariant,
                Elfreloc1:        elfreloc1,
+               ElfrelocSize:     24,
                Elfsetupplt:      elfsetupplt,
                Gentext:          gentext,
                Machoreloc1:      machoreloc1,

diff --git a/src/cmd/link/internal/sym/segment.go b/src/cmd/link/internal/sym/segment.go
index 9b49c62dda08812047e19b01b868c54ade153dcc..97853b9355f4e908c69436c1d6741b92a5d7d982 100644 (file)
--- a/src/cmd/link/internal/sym/segment.go
+++ b/src/cmd/link/internal/sym/segment.go
@@ -55,6 +55,12 @@ type Section struct {
        Elfsect interface{} // an *ld.ElfShdr
        Reloff  uint64
        Rellen  uint64
-       Sym     LoaderSym // symbol for the section, if any
-       Index   uint16    // each section has a unique index, used internally
+       // Relcount is the number of *host* relocations applied to this section
+       // (when external linking).
+       // Incremented atomically on multiple goroutines.
+       // Note: this may differ from number of Go relocations, as one Go relocation
+       // may turn into multiple host relocations.
+       Relcount uint32
+       Sym      LoaderSym // symbol for the section, if any
+       Index    uint16    // each section has a unique index, used internally
 }