From ad642727247383079c8546ca365172859641a800 Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Tue, 25 Aug 2020 12:34:02 -0400 Subject: [PATCH] runtime: rename pageAlloc receiver The history of pageAlloc using 's' as a receiver are lost to the depths of time (perhaps it used to be called summary?), but it doesn't make much sense anymore. Rename it to 'p'. Generated with: $ cd src/runtime $ grep -R -b "func (s \*pageAlloc" . | awk -F : '{ print $1 ":#" $2+6 }' | xargs -n 1 -I {} env GOROOT=$(pwd)/../../ gorename -offset {} -to p -v $ grep -R -b "func (s \*pageAlloc" . | awk -F : '{ print $1 ":#" $2+6 }' | xargs -n 1 -I {} env GOROOT=$(pwd)/../../ GOARCH=386 gorename -offset {} -to p -v $ GOROOT=$(pwd)/../../ gorename -offset mpagecache.go:#2397 -to p -v ($2+6 to advance past "func (".) Plus manual comment fixups. Change-Id: I2d521a1cbf6ebe2ef6aae92e654bfc33c63d1aa9 Reviewed-on: https://go-review.googlesource.com/c/go/+/250517 Trust: Michael Pratt Run-TryBot: Michael Pratt TryBot-Result: Go Bot Reviewed-by: Michael Knyszek --- src/runtime/mgcscavenge.go | 102 ++++++++-------- src/runtime/mpagealloc.go | 200 ++++++++++++++++---------------- src/runtime/mpagealloc_32bit.go | 14 +-- src/runtime/mpagealloc_64bit.go | 30 ++--- src/runtime/mpagecache.go | 42 +++---- 5 files changed, 194 insertions(+), 194 deletions(-) diff --git a/src/runtime/mgcscavenge.go b/src/runtime/mgcscavenge.go index 9d6f551768..34646828e5 100644 --- a/src/runtime/mgcscavenge.go +++ b/src/runtime/mgcscavenge.go @@ -390,13 +390,13 @@ func bgscavenge(c chan int) { // // Returns the amount of memory scavenged in bytes. // -// s.mheapLock must be held, but may be temporarily released if +// p.mheapLock must be held, but may be temporarily released if // mayUnlock == true. // -// Must run on the system stack because s.mheapLock must be held. +// Must run on the system stack because p.mheapLock must be held. // //go:systemstack -func (s *pageAlloc) scavenge(nbytes uintptr, mayUnlock bool) uintptr { +func (p *pageAlloc) scavenge(nbytes uintptr, mayUnlock bool) uintptr { var ( addrs addrRange gen uint32 @@ -404,17 +404,17 @@ func (s *pageAlloc) scavenge(nbytes uintptr, mayUnlock bool) uintptr { released := uintptr(0) for released < nbytes { if addrs.size() == 0 { - if addrs, gen = s.scavengeReserve(); addrs.size() == 0 { + if addrs, gen = p.scavengeReserve(); addrs.size() == 0 { break } } - r, a := s.scavengeOne(addrs, nbytes-released, mayUnlock) + r, a := p.scavengeOne(addrs, nbytes-released, mayUnlock) released += r addrs = a } // Only unreserve the space which hasn't been scavenged or searched // to ensure we always make progress. - s.scavengeUnreserve(addrs, gen) + p.scavengeUnreserve(addrs, gen) return released } @@ -440,46 +440,46 @@ func printScavTrace(gen uint32, released uintptr, forced bool) { // scavengeStartGen starts a new scavenge generation, resetting // the scavenger's search space to the full in-use address space. // -// s.mheapLock must be held. +// p.mheapLock must be held. // -// Must run on the system stack because s.mheapLock must be held. +// Must run on the system stack because p.mheapLock must be held. // //go:systemstack -func (s *pageAlloc) scavengeStartGen() { +func (p *pageAlloc) scavengeStartGen() { if debug.scavtrace > 0 { - printScavTrace(s.scav.gen, s.scav.released, false) + printScavTrace(p.scav.gen, p.scav.released, false) } - s.inUse.cloneInto(&s.scav.inUse) + p.inUse.cloneInto(&p.scav.inUse) // Pick the new starting address for the scavenger cycle. var startAddr offAddr - if s.scav.scavLWM.lessThan(s.scav.freeHWM) { + if p.scav.scavLWM.lessThan(p.scav.freeHWM) { // The "free" high watermark exceeds the "scavenged" low watermark, // so there are free scavengable pages in parts of the address space // that the scavenger already searched, the high watermark being the // highest one. Pick that as our new starting point to ensure we // see those pages. - startAddr = s.scav.freeHWM + startAddr = p.scav.freeHWM } else { // The "free" high watermark does not exceed the "scavenged" low // watermark. This means the allocator didn't free any memory in // the range we scavenged last cycle, so we might as well continue // scavenging from where we were. - startAddr = s.scav.scavLWM + startAddr = p.scav.scavLWM } - s.scav.inUse.removeGreaterEqual(startAddr.addr()) + p.scav.inUse.removeGreaterEqual(startAddr.addr()) - // reservationBytes may be zero if s.inUse.totalBytes is small, or if + // reservationBytes may be zero if p.inUse.totalBytes is small, or if // scavengeReservationShards is large. This case is fine as the scavenger // will simply be turned off, but it does mean that scavengeReservationShards, // in concert with pallocChunkBytes, dictates the minimum heap size at which // the scavenger triggers. In practice this minimum is generally less than an // arena in size, so virtually every heap has the scavenger on. - s.scav.reservationBytes = alignUp(s.inUse.totalBytes, pallocChunkBytes) / scavengeReservationShards - s.scav.gen++ - s.scav.released = 0 - s.scav.freeHWM = minOffAddr - s.scav.scavLWM = maxOffAddr + p.scav.reservationBytes = alignUp(p.inUse.totalBytes, pallocChunkBytes) / scavengeReservationShards + p.scav.gen++ + p.scav.released = 0 + p.scav.freeHWM = minOffAddr + p.scav.scavLWM = maxOffAddr } // scavengeReserve reserves a contiguous range of the address space @@ -489,19 +489,19 @@ func (s *pageAlloc) scavengeStartGen() { // // Returns the reserved range and the scavenge generation number for it. // -// s.mheapLock must be held. +// p.mheapLock must be held. // -// Must run on the system stack because s.mheapLock must be held. +// Must run on the system stack because p.mheapLock must be held. // //go:systemstack -func (s *pageAlloc) scavengeReserve() (addrRange, uint32) { +func (p *pageAlloc) scavengeReserve() (addrRange, uint32) { // Start by reserving the minimum. - r := s.scav.inUse.removeLast(s.scav.reservationBytes) + r := p.scav.inUse.removeLast(p.scav.reservationBytes) // Return early if the size is zero; we don't want to use // the bogus address below. if r.size() == 0 { - return r, s.scav.gen + return r, p.scav.gen } // The scavenger requires that base be aligned to a @@ -511,27 +511,27 @@ func (s *pageAlloc) scavengeReserve() (addrRange, uint32) { newBase := alignDown(r.base.addr(), pallocChunkBytes) // Remove from inUse however much extra we just pulled out. - s.scav.inUse.removeGreaterEqual(newBase) + p.scav.inUse.removeGreaterEqual(newBase) r.base = offAddr{newBase} - return r, s.scav.gen + return r, p.scav.gen } // scavengeUnreserve returns an unscavenged portion of a range that was // previously reserved with scavengeReserve. // -// s.mheapLock must be held. +// p.mheapLock must be held. // -// Must run on the system stack because s.mheapLock must be held. +// Must run on the system stack because p.mheapLock must be held. // //go:systemstack -func (s *pageAlloc) scavengeUnreserve(r addrRange, gen uint32) { - if r.size() == 0 || gen != s.scav.gen { +func (p *pageAlloc) scavengeUnreserve(r addrRange, gen uint32) { + if r.size() == 0 || gen != p.scav.gen { return } if r.base.addr()%pallocChunkBytes != 0 { throw("unreserving unaligned region") } - s.scav.inUse.add(r) + p.scav.inUse.add(r) } // scavengeOne walks over address range work until it finds @@ -545,13 +545,13 @@ func (s *pageAlloc) scavengeUnreserve(r addrRange, gen uint32) { // // work's base address must be aligned to pallocChunkBytes. // -// s.mheapLock must be held, but may be temporarily released if +// p.mheapLock must be held, but may be temporarily released if // mayUnlock == true. // -// Must run on the system stack because s.mheapLock must be held. +// Must run on the system stack because p.mheapLock must be held. // //go:systemstack -func (s *pageAlloc) scavengeOne(work addrRange, max uintptr, mayUnlock bool) (uintptr, addrRange) { +func (p *pageAlloc) scavengeOne(work addrRange, max uintptr, mayUnlock bool) (uintptr, addrRange) { // Defensively check if we've recieved an empty address range. // If so, just return. if work.size() == 0 { @@ -586,12 +586,12 @@ func (s *pageAlloc) scavengeOne(work addrRange, max uintptr, mayUnlock bool) (ui // Helpers for locking and unlocking only if mayUnlock == true. lockHeap := func() { if mayUnlock { - lock(s.mheapLock) + lock(p.mheapLock) } } unlockHeap := func() { if mayUnlock { - unlock(s.mheapLock) + unlock(p.mheapLock) } } @@ -602,14 +602,14 @@ func (s *pageAlloc) scavengeOne(work addrRange, max uintptr, mayUnlock bool) (ui // by subtracting 1. maxAddr := work.limit.addr() - 1 maxChunk := chunkIndex(maxAddr) - if s.summary[len(s.summary)-1][maxChunk].max() >= uint(minPages) { + if p.summary[len(p.summary)-1][maxChunk].max() >= uint(minPages) { // We only bother looking for a candidate if there at least // minPages free pages at all. - base, npages := s.chunkOf(maxChunk).findScavengeCandidate(chunkPageIndex(maxAddr), minPages, maxPages) + base, npages := p.chunkOf(maxChunk).findScavengeCandidate(chunkPageIndex(maxAddr), minPages, maxPages) // If we found something, scavenge it and return! if npages != 0 { - work.limit = offAddr{s.scavengeRangeLocked(maxChunk, base, npages)} + work.limit = offAddr{p.scavengeRangeLocked(maxChunk, base, npages)} return uintptr(npages) * pageSize, work } } @@ -631,7 +631,7 @@ func (s *pageAlloc) scavengeOne(work addrRange, max uintptr, mayUnlock bool) (ui // that's fine. We're being optimistic anyway. // Check quickly if there are enough free pages at all. - if s.summary[len(s.summary)-1][i].max() < uint(minPages) { + if p.summary[len(p.summary)-1][i].max() < uint(minPages) { continue } @@ -641,7 +641,7 @@ func (s *pageAlloc) scavengeOne(work addrRange, max uintptr, mayUnlock bool) (ui // avoid races with heap growth. It may or may not be possible to also // see a nil pointer in this case if we do race with heap growth, but // just defensively ignore the nils. This operation is optimistic anyway. - l2 := (*[1 << pallocChunksL2Bits]pallocData)(atomic.Loadp(unsafe.Pointer(&s.chunks[i.l1()]))) + l2 := (*[1 << pallocChunksL2Bits]pallocData)(atomic.Loadp(unsafe.Pointer(&p.chunks[i.l1()]))) if l2 != nil && l2[i.l2()].hasScavengeCandidate(minPages) { return i, true } @@ -670,10 +670,10 @@ func (s *pageAlloc) scavengeOne(work addrRange, max uintptr, mayUnlock bool) (ui } // Find, verify, and scavenge if we can. - chunk := s.chunkOf(candidateChunkIdx) + chunk := p.chunkOf(candidateChunkIdx) base, npages := chunk.findScavengeCandidate(pallocChunkPages-1, minPages, maxPages) if npages > 0 { - work.limit = offAddr{s.scavengeRangeLocked(candidateChunkIdx, base, npages)} + work.limit = offAddr{p.scavengeRangeLocked(candidateChunkIdx, base, npages)} return uintptr(npages) * pageSize, work } @@ -690,21 +690,21 @@ func (s *pageAlloc) scavengeOne(work addrRange, max uintptr, mayUnlock bool) (ui // // Returns the base address of the scavenged region. // -// s.mheapLock must be held. -func (s *pageAlloc) scavengeRangeLocked(ci chunkIdx, base, npages uint) uintptr { - s.chunkOf(ci).scavenged.setRange(base, npages) +// p.mheapLock must be held. +func (p *pageAlloc) scavengeRangeLocked(ci chunkIdx, base, npages uint) uintptr { + p.chunkOf(ci).scavenged.setRange(base, npages) // Compute the full address for the start of the range. addr := chunkBase(ci) + uintptr(base)*pageSize // Update the scavenge low watermark. - if oAddr := (offAddr{addr}); oAddr.lessThan(s.scav.scavLWM) { - s.scav.scavLWM = oAddr + if oAddr := (offAddr{addr}); oAddr.lessThan(p.scav.scavLWM) { + p.scav.scavLWM = oAddr } // Only perform the actual scavenging if we're not in a test. // It's dangerous to do so otherwise. - if s.test { + if p.test { return addr } sysUnused(unsafe.Pointer(addr), uintptr(npages)*pageSize) diff --git a/src/runtime/mpagealloc.go b/src/runtime/mpagealloc.go index c90a6378bd..560babed03 100644 --- a/src/runtime/mpagealloc.go +++ b/src/runtime/mpagealloc.go @@ -299,7 +299,7 @@ type pageAlloc struct { test bool } -func (s *pageAlloc) init(mheapLock *mutex, sysStat *uint64) { +func (p *pageAlloc) init(mheapLock *mutex, sysStat *uint64) { if levelLogPages[0] > logMaxPackedValue { // We can't represent 1< s.end { - s.end = end + if end > p.end { + p.end = end } // Note that [base, limit) will never overlap with any existing // range inUse because grow only ever adds never-used memory // regions to the page allocator. - s.inUse.add(makeAddrRange(base, limit)) + p.inUse.add(makeAddrRange(base, limit)) // A grow operation is a lot like a free operation, so if our - // chunk ends up below s.searchAddr, update s.searchAddr to the + // chunk ends up below p.searchAddr, update p.searchAddr to the // new address, just like in free. - if b := (offAddr{base}); b.lessThan(s.searchAddr) { - s.searchAddr = b + if b := (offAddr{base}); b.lessThan(p.searchAddr) { + p.searchAddr = b } // Add entries into chunks, which is sparse, if needed. Then, @@ -387,21 +387,21 @@ func (s *pageAlloc) grow(base, size uintptr) { // Newly-grown memory is always considered scavenged. // Set all the bits in the scavenged bitmaps high. for c := chunkIndex(base); c < chunkIndex(limit); c++ { - if s.chunks[c.l1()] == nil { + if p.chunks[c.l1()] == nil { // Create the necessary l2 entry. // // Store it atomically to avoid races with readers which // don't acquire the heap lock. - r := sysAlloc(unsafe.Sizeof(*s.chunks[0]), s.sysStat) - atomic.StorepNoWB(unsafe.Pointer(&s.chunks[c.l1()]), r) + r := sysAlloc(unsafe.Sizeof(*p.chunks[0]), p.sysStat) + atomic.StorepNoWB(unsafe.Pointer(&p.chunks[c.l1()]), r) } - s.chunkOf(c).scavenged.setRange(0, pallocChunkPages) + p.chunkOf(c).scavenged.setRange(0, pallocChunkPages) } // Update summaries accordingly. The grow acts like a free, so // we need to ensure this newly-free memory is visible in the // summaries. - s.update(base, size/pageSize, true, false) + p.update(base, size/pageSize, true, false) } // update updates heap metadata. It must be called each time the bitmap @@ -411,8 +411,8 @@ func (s *pageAlloc) grow(base, size uintptr) { // a contiguous allocation or free between addr and addr+npages. alloc indicates // whether the operation performed was an allocation or a free. // -// s.mheapLock must be held. -func (s *pageAlloc) update(base, npages uintptr, contig, alloc bool) { +// p.mheapLock must be held. +func (p *pageAlloc) update(base, npages uintptr, contig, alloc bool) { // base, limit, start, and end are inclusive. limit := base + npages*pageSize - 1 sc, ec := chunkIndex(base), chunkIndex(limit) @@ -421,23 +421,23 @@ func (s *pageAlloc) update(base, npages uintptr, contig, alloc bool) { if sc == ec { // Fast path: the allocation doesn't span more than one chunk, // so update this one and if the summary didn't change, return. - x := s.summary[len(s.summary)-1][sc] - y := s.chunkOf(sc).summarize() + x := p.summary[len(p.summary)-1][sc] + y := p.chunkOf(sc).summarize() if x == y { return } - s.summary[len(s.summary)-1][sc] = y + p.summary[len(p.summary)-1][sc] = y } else if contig { // Slow contiguous path: the allocation spans more than one chunk // and at least one summary is guaranteed to change. - summary := s.summary[len(s.summary)-1] + summary := p.summary[len(p.summary)-1] // Update the summary for chunk sc. - summary[sc] = s.chunkOf(sc).summarize() + summary[sc] = p.chunkOf(sc).summarize() // Update the summaries for chunks in between, which are // either totally allocated or freed. - whole := s.summary[len(s.summary)-1][sc+1 : ec] + whole := p.summary[len(p.summary)-1][sc+1 : ec] if alloc { // Should optimize into a memclr. for i := range whole { @@ -450,22 +450,22 @@ func (s *pageAlloc) update(base, npages uintptr, contig, alloc bool) { } // Update the summary for chunk ec. - summary[ec] = s.chunkOf(ec).summarize() + summary[ec] = p.chunkOf(ec).summarize() } else { // Slow general path: the allocation spans more than one chunk // and at least one summary is guaranteed to change. // // We can't assume a contiguous allocation happened, so walk over // every chunk in the range and manually recompute the summary. - summary := s.summary[len(s.summary)-1] + summary := p.summary[len(p.summary)-1] for c := sc; c <= ec; c++ { - summary[c] = s.chunkOf(c).summarize() + summary[c] = p.chunkOf(c).summarize() } } // Walk up the radix tree and update the summaries appropriately. changed := true - for l := len(s.summary) - 2; l >= 0 && changed; l-- { + for l := len(p.summary) - 2; l >= 0 && changed; l-- { // Update summaries at level l from summaries at level l+1. changed = false @@ -479,12 +479,12 @@ func (s *pageAlloc) update(base, npages uintptr, contig, alloc bool) { // Iterate over each block, updating the corresponding summary in the less-granular level. for i := lo; i < hi; i++ { - children := s.summary[l+1][i<= s.end { + if chunkIndex(p.searchAddr.addr()) >= p.end { return 0, 0 } // If npages has a chance of fitting in the chunk where the searchAddr is, // search it directly. searchAddr := minOffAddr - if pallocChunkPages-chunkPageIndex(s.searchAddr.addr()) >= uint(npages) { + if pallocChunkPages-chunkPageIndex(p.searchAddr.addr()) >= uint(npages) { // npages is guaranteed to be no greater than pallocChunkPages here. - i := chunkIndex(s.searchAddr.addr()) - if max := s.summary[len(s.summary)-1][i].max(); max >= uint(npages) { - j, searchIdx := s.chunkOf(i).find(npages, chunkPageIndex(s.searchAddr.addr())) + i := chunkIndex(p.searchAddr.addr()) + if max := p.summary[len(p.summary)-1][i].max(); max >= uint(npages) { + j, searchIdx := p.chunkOf(i).find(npages, chunkPageIndex(p.searchAddr.addr())) if j == ^uint(0) { print("runtime: max = ", max, ", npages = ", npages, "\n") - print("runtime: searchIdx = ", chunkPageIndex(s.searchAddr.addr()), ", s.searchAddr = ", hex(s.searchAddr.addr()), "\n") + print("runtime: searchIdx = ", chunkPageIndex(p.searchAddr.addr()), ", p.searchAddr = ", hex(p.searchAddr.addr()), "\n") throw("bad summary data") } addr = chunkBase(i) + uintptr(j)*pageSize @@ -813,7 +813,7 @@ func (s *pageAlloc) alloc(npages uintptr) (addr uintptr, scav uintptr) { } // We failed to use a searchAddr for one reason or another, so try // the slow path. - addr, searchAddr = s.find(npages) + addr, searchAddr = p.find(npages) if addr == 0 { if npages == 1 { // We failed to find a single free page, the smallest unit @@ -821,41 +821,41 @@ func (s *pageAlloc) alloc(npages uintptr) (addr uintptr, scav uintptr) { // exhausted. Otherwise, the heap still might have free // space in it, just not enough contiguous space to // accommodate npages. - s.searchAddr = maxSearchAddr + p.searchAddr = maxSearchAddr } return 0, 0 } Found: // Go ahead and actually mark the bits now that we have an address. - scav = s.allocRange(addr, npages) + scav = p.allocRange(addr, npages) // If we found a higher searchAddr, we know that all the // heap memory before that searchAddr in an offset address space is - // allocated, so bump s.searchAddr up to the new one. - if s.searchAddr.lessThan(searchAddr) { - s.searchAddr = searchAddr + // allocated, so bump p.searchAddr up to the new one. + if p.searchAddr.lessThan(searchAddr) { + p.searchAddr = searchAddr } return addr, scav } // free returns npages worth of memory starting at base back to the page heap. // -// s.mheapLock must be held. -func (s *pageAlloc) free(base, npages uintptr) { - // If we're freeing pages below the s.searchAddr, update searchAddr. - if b := (offAddr{base}); b.lessThan(s.searchAddr) { - s.searchAddr = b +// p.mheapLock must be held. +func (p *pageAlloc) free(base, npages uintptr) { + // If we're freeing pages below the p.searchAddr, update searchAddr. + if b := (offAddr{base}); b.lessThan(p.searchAddr) { + p.searchAddr = b } // Update the free high watermark for the scavenger. limit := base + npages*pageSize - 1 - if offLimit := (offAddr{limit}); s.scav.freeHWM.lessThan(offLimit) { - s.scav.freeHWM = offLimit + if offLimit := (offAddr{limit}); p.scav.freeHWM.lessThan(offLimit) { + p.scav.freeHWM = offLimit } if npages == 1 { // Fast path: we're clearing a single bit, and we know exactly // where it is, so mark it directly. i := chunkIndex(base) - s.chunkOf(i).free1(chunkPageIndex(base)) + p.chunkOf(i).free1(chunkPageIndex(base)) } else { // Slow path: we're clearing more bits so we may need to iterate. sc, ec := chunkIndex(base), chunkIndex(limit) @@ -863,17 +863,17 @@ func (s *pageAlloc) free(base, npages uintptr) { if sc == ec { // The range doesn't cross any chunk boundaries. - s.chunkOf(sc).free(si, ei+1-si) + p.chunkOf(sc).free(si, ei+1-si) } else { // The range crosses at least one chunk boundary. - s.chunkOf(sc).free(si, pallocChunkPages-si) + p.chunkOf(sc).free(si, pallocChunkPages-si) for c := sc + 1; c < ec; c++ { - s.chunkOf(c).freeAll() + p.chunkOf(c).freeAll() } - s.chunkOf(ec).free(0, ei+1) + p.chunkOf(ec).free(0, ei+1) } } - s.update(base, npages, true, false) + p.update(base, npages, true, false) } const ( diff --git a/src/runtime/mpagealloc_32bit.go b/src/runtime/mpagealloc_32bit.go index 90f1e54d6c..331dadade9 100644 --- a/src/runtime/mpagealloc_32bit.go +++ b/src/runtime/mpagealloc_32bit.go @@ -60,7 +60,7 @@ var levelLogPages = [summaryLevels]uint{ } // See mpagealloc_64bit.go for details. -func (s *pageAlloc) sysInit() { +func (p *pageAlloc) sysInit() { // Calculate how much memory all our entries will take up. // // This should be around 12 KiB or less. @@ -76,7 +76,7 @@ func (s *pageAlloc) sysInit() { throw("failed to reserve page summary memory") } // There isn't much. Just map it and mark it as used immediately. - sysMap(reservation, totalSize, s.sysStat) + sysMap(reservation, totalSize, p.sysStat) sysUsed(reservation, totalSize) // Iterate over the reservation and cut it up into slices. @@ -88,29 +88,29 @@ func (s *pageAlloc) sysInit() { // Put this reservation into a slice. sl := notInHeapSlice{(*notInHeap)(reservation), 0, entries} - s.summary[l] = *(*[]pallocSum)(unsafe.Pointer(&sl)) + p.summary[l] = *(*[]pallocSum)(unsafe.Pointer(&sl)) reservation = add(reservation, uintptr(entries)*pallocSumBytes) } } // See mpagealloc_64bit.go for details. -func (s *pageAlloc) sysGrow(base, limit uintptr) { +func (p *pageAlloc) sysGrow(base, limit uintptr) { if base%pallocChunkBytes != 0 || limit%pallocChunkBytes != 0 { print("runtime: base = ", hex(base), ", limit = ", hex(limit), "\n") throw("sysGrow bounds not aligned to pallocChunkBytes") } // Walk up the tree and update the summary slices. - for l := len(s.summary) - 1; l >= 0; l-- { + for l := len(p.summary) - 1; l >= 0; l-- { // Figure out what part of the summary array this new address space needs. // Note that we need to align the ranges to the block width (1< len(s.summary[l]) { - s.summary[l] = s.summary[l][:hi] + if hi > len(p.summary[l]) { + p.summary[l] = p.summary[l][:hi] } } } diff --git a/src/runtime/mpagealloc_64bit.go b/src/runtime/mpagealloc_64bit.go index a1691ba802..ffacb46c18 100644 --- a/src/runtime/mpagealloc_64bit.go +++ b/src/runtime/mpagealloc_64bit.go @@ -67,7 +67,7 @@ var levelLogPages = [summaryLevels]uint{ // sysInit performs architecture-dependent initialization of fields // in pageAlloc. pageAlloc should be uninitialized except for sysStat // if any runtime statistic should be updated. -func (s *pageAlloc) sysInit() { +func (p *pageAlloc) sysInit() { // Reserve memory for each level. This will get mapped in // as R/W by setArenas. for l, shift := range levelShift { @@ -82,21 +82,21 @@ func (s *pageAlloc) sysInit() { // Put this reservation into a slice. sl := notInHeapSlice{(*notInHeap)(r), 0, entries} - s.summary[l] = *(*[]pallocSum)(unsafe.Pointer(&sl)) + p.summary[l] = *(*[]pallocSum)(unsafe.Pointer(&sl)) } } // sysGrow performs architecture-dependent operations on heap // growth for the page allocator, such as mapping in new memory // for summaries. It also updates the length of the slices in -// s.summary. +// [.summary. // // base is the base of the newly-added heap memory and limit is // the first address past the end of the newly-added heap memory. // Both must be aligned to pallocChunkBytes. // -// The caller must update s.start and s.end after calling sysGrow. -func (s *pageAlloc) sysGrow(base, limit uintptr) { +// The caller must update p.start and p.end after calling sysGrow. +func (p *pageAlloc) sysGrow(base, limit uintptr) { if base%pallocChunkBytes != 0 || limit%pallocChunkBytes != 0 { print("runtime: base = ", hex(base), ", limit = ", hex(limit), "\n") throw("sysGrow bounds not aligned to pallocChunkBytes") @@ -111,12 +111,12 @@ func (s *pageAlloc) sysGrow(base, limit uintptr) { } // summaryRangeToSumAddrRange converts a range of indices in any - // level of s.summary into page-aligned addresses which cover that + // level of p.summary into page-aligned addresses which cover that // range of indices. summaryRangeToSumAddrRange := func(level, sumIdxBase, sumIdxLimit int) addrRange { baseOffset := alignDown(uintptr(sumIdxBase)*pallocSumBytes, physPageSize) limitOffset := alignUp(uintptr(sumIdxLimit)*pallocSumBytes, physPageSize) - base := unsafe.Pointer(&s.summary[level][0]) + base := unsafe.Pointer(&p.summary[level][0]) return addrRange{ offAddr{uintptr(add(base, baseOffset))}, offAddr{uintptr(add(base, limitOffset))}, @@ -140,10 +140,10 @@ func (s *pageAlloc) sysGrow(base, limit uintptr) { // // This will be used to look at what memory in the summary array is already // mapped before and after this new range. - inUseIndex := s.inUse.findSucc(base) + inUseIndex := p.inUse.findSucc(base) // Walk up the radix tree and map summaries in as needed. - for l := range s.summary { + for l := range p.summary { // Figure out what part of the summary array this new address space needs. needIdxBase, needIdxLimit := addrRangeToSummaryRange(l, makeAddrRange(base, limit)) @@ -151,8 +151,8 @@ func (s *pageAlloc) sysGrow(base, limit uintptr) { // we get tight bounds checks on at least the top bound. // // We must do this regardless of whether we map new memory. - if needIdxLimit > len(s.summary[l]) { - s.summary[l] = s.summary[l][:needIdxLimit] + if needIdxLimit > len(p.summary[l]) { + p.summary[l] = p.summary[l][:needIdxLimit] } // Compute the needed address range in the summary array for level l. @@ -163,10 +163,10 @@ func (s *pageAlloc) sysGrow(base, limit uintptr) { // for mapping. prune's invariants are guaranteed by the fact that this // function will never be asked to remap the same memory twice. if inUseIndex > 0 { - need = need.subtract(addrRangeToSumAddrRange(l, s.inUse.ranges[inUseIndex-1])) + need = need.subtract(addrRangeToSumAddrRange(l, p.inUse.ranges[inUseIndex-1])) } - if inUseIndex < len(s.inUse.ranges) { - need = need.subtract(addrRangeToSumAddrRange(l, s.inUse.ranges[inUseIndex])) + if inUseIndex < len(p.inUse.ranges) { + need = need.subtract(addrRangeToSumAddrRange(l, p.inUse.ranges[inUseIndex])) } // It's possible that after our pruning above, there's nothing new to map. if need.size() == 0 { @@ -174,7 +174,7 @@ func (s *pageAlloc) sysGrow(base, limit uintptr) { } // Map and commit need. - sysMap(unsafe.Pointer(need.base.addr()), need.size(), s.sysStat) + sysMap(unsafe.Pointer(need.base.addr()), need.size(), p.sysStat) sysUsed(unsafe.Pointer(need.base.addr()), need.size()) } } diff --git a/src/runtime/mpagecache.go b/src/runtime/mpagecache.go index 683a997136..5f76501a1c 100644 --- a/src/runtime/mpagecache.go +++ b/src/runtime/mpagecache.go @@ -71,8 +71,8 @@ func (c *pageCache) allocN(npages uintptr) (uintptr, uintptr) { // into s. Then, it clears the cache, such that empty returns // true. // -// s.mheapLock must be held or the world must be stopped. -func (c *pageCache) flush(s *pageAlloc) { +// p.mheapLock must be held or the world must be stopped. +func (c *pageCache) flush(p *pageAlloc) { if c.empty() { return } @@ -83,18 +83,18 @@ func (c *pageCache) flush(s *pageAlloc) { // slower, safer thing by iterating over each bit individually. for i := uint(0); i < 64; i++ { if c.cache&(1<= s.end { + if chunkIndex(p.searchAddr.addr()) >= p.end { return pageCache{} } c := pageCache{} - ci := chunkIndex(s.searchAddr.addr()) // chunk index - if s.summary[len(s.summary)-1][ci] != 0 { + ci := chunkIndex(p.searchAddr.addr()) // chunk index + if p.summary[len(p.summary)-1][ci] != 0 { // Fast path: there's free pages at or near the searchAddr address. - chunk := s.chunkOf(ci) - j, _ := chunk.find(1, chunkPageIndex(s.searchAddr.addr())) + chunk := p.chunkOf(ci) + j, _ := chunk.find(1, chunkPageIndex(p.searchAddr.addr())) if j == ^uint(0) { throw("bad summary data") } @@ -126,15 +126,15 @@ func (s *pageAlloc) allocToCache() pageCache { } else { // Slow path: the searchAddr address had nothing there, so go find // the first free page the slow way. - addr, _ := s.find(1) + addr, _ := p.find(1) if addr == 0 { // We failed to find adequate free space, so mark the searchAddr as OoM // and return an empty pageCache. - s.searchAddr = maxSearchAddr + p.searchAddr = maxSearchAddr return pageCache{} } ci := chunkIndex(addr) - chunk := s.chunkOf(ci) + chunk := p.chunkOf(ci) c = pageCache{ base: alignDown(addr, 64*pageSize), cache: ^chunk.pages64(chunkPageIndex(addr)), @@ -143,19 +143,19 @@ func (s *pageAlloc) allocToCache() pageCache { } // Set the bits as allocated and clear the scavenged bits. - s.allocRange(c.base, pageCachePages) + p.allocRange(c.base, pageCachePages) // Update as an allocation, but note that it's not contiguous. - s.update(c.base, pageCachePages, false, true) + p.update(c.base, pageCachePages, false, true) // Set the search address to the last page represented by the cache. // Since all of the pages in this block are going to the cache, and we // searched for the first free page, we can confidently start at the // next page. // - // However, s.searchAddr is not allowed to point into unmapped heap memory + // However, p.searchAddr is not allowed to point into unmapped heap memory // unless it is maxSearchAddr, so make it the last page as opposed to // the page after. - s.searchAddr = offAddr{c.base + pageSize*(pageCachePages-1)} + p.searchAddr = offAddr{c.base + pageSize*(pageCachePages-1)} return c } -- 2.48.1