From 3024785b929cd8a740da87e1c24aef2c9b30e019 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Wed, 18 Jun 2025 14:50:23 -0700 Subject: [PATCH] cmd/compile,runtime: remember idx+len for bounds check failure with less code Currently we must put the index and length into specific registers so we can call into the runtime to report a bounds check failure. So a typical bounds check call is something like: MOVD R3, R0 MOVD R7, R1 CALL runtime.panicIndex or, if for instance the index is constant, MOVD $7, R0 MOVD R9, R1 CALL runtime.panicIndex Sometimes the MOVD can be avoided, if the value happens to be in the right register already. But that's not terribly common, and doesn't work at all for constants. Let's get rid of those MOVD instructions. They pollute the instruction cache and are almost never executed. Instead, we'll encode in a PCDATA table where the runtime should find the index and length. The table encodes, for each index and length, whether it is a constant or in a register, and which register or constant it is. That way, we can avoid all those useless MOVDs. Instead, we can figure out the index and length at runtime. This makes the bounds panic path slower, but that's a good tradeoff. We can encode registers 0-15 and constants 0-31. Anything outside that range still needs to use an explicit instruction. This CL is the foundation, followon CLs will move each architecture to the new strategy. Change-Id: I705c511e546e6aac59fed922a8eaed4585e96820 Reviewed-on: https://go-review.googlesource.com/c/go/+/682396 Reviewed-by: Michael Knyszek Reviewed-by: David Chase LUCI-TryBot-Result: Go LUCI --- src/cmd/compile/internal/ir/symtab.go | 2 + src/cmd/compile/internal/ssa/_gen/rulegen.go | 18 +++- src/cmd/compile/internal/ssa/check.go | 3 + src/cmd/compile/internal/ssa/op.go | 49 +++++++++++ src/cmd/compile/internal/ssa/rewrite.go | 29 ++++++ src/cmd/compile/internal/ssagen/ssa.go | 2 + src/internal/abi/bounds.go | 92 +++++++++++++++++++ src/internal/abi/symtab.go | 1 + src/runtime/panic.go | 93 ++++++++++++++++++++ 9 files changed, 287 insertions(+), 2 deletions(-) diff --git a/src/cmd/compile/internal/ir/symtab.go b/src/cmd/compile/internal/ir/symtab.go index e2da710f02..ee0f52fbf3 100644 --- a/src/cmd/compile/internal/ir/symtab.go +++ b/src/cmd/compile/internal/ir/symtab.go @@ -37,6 +37,8 @@ type symsStruct struct { Msanmove *obj.LSym Newobject *obj.LSym Newproc *obj.LSym + PanicBounds *obj.LSym + PanicExtend *obj.LSym Panicdivide *obj.LSym Panicshift *obj.LSym PanicdottypeE *obj.LSym diff --git a/src/cmd/compile/internal/ssa/_gen/rulegen.go b/src/cmd/compile/internal/ssa/_gen/rulegen.go index 3854d58b7f..d21647ad6d 100644 --- a/src/cmd/compile/internal/ssa/_gen/rulegen.go +++ b/src/cmd/compile/internal/ssa/_gen/rulegen.go @@ -540,6 +540,13 @@ func (u *unusedInspector) node(node ast.Node) { } } case *ast.BasicLit: + case *ast.CompositeLit: + for _, e := range node.Elts { + u.node(e) + } + case *ast.KeyValueExpr: + u.node(node.Key) + u.node(node.Value) case *ast.ValueSpec: u.exprs(node.Values) default: @@ -1431,7 +1438,8 @@ func parseValue(val string, arch arch, loc string) (op opData, oparch, typ, auxi func opHasAuxInt(op opData) bool { switch op.aux { case "Bool", "Int8", "Int16", "Int32", "Int64", "Int128", "UInt8", "Float32", "Float64", - "SymOff", "CallOff", "SymValAndOff", "TypSize", "ARM64BitField", "FlagConstant", "CCop": + "SymOff", "CallOff", "SymValAndOff", "TypSize", "ARM64BitField", "FlagConstant", "CCop", + "PanicBoundsC", "PanicBoundsCC": return true } return false @@ -1440,7 +1448,7 @@ func opHasAuxInt(op opData) bool { func opHasAux(op opData) bool { switch op.aux { case "String", "Sym", "SymOff", "Call", "CallOff", "SymValAndOff", "Typ", "TypSize", - "S390XCCMask", "S390XRotateParams": + "S390XCCMask", "S390XRotateParams", "PanicBoundsC", "PanicBoundsCC": return true } return false @@ -1795,6 +1803,10 @@ func (op opData) auxType() string { return "s390x.CCMask" case "S390XRotateParams": return "s390x.RotateParams" + case "PanicBoundsC": + return "PanicBoundsC" + case "PanicBoundsCC": + return "PanicBoundsCC" default: return "invalid" } @@ -1835,6 +1847,8 @@ func (op opData) auxIntType() string { return "flagConstant" case "ARM64BitField": return "arm64BitField" + case "PanicBoundsC", "PanicBoundsCC": + return "int64" default: return "invalid" } diff --git a/src/cmd/compile/internal/ssa/check.go b/src/cmd/compile/internal/ssa/check.go index cb6788cd95..f33c9bc87b 100644 --- a/src/cmd/compile/internal/ssa/check.go +++ b/src/cmd/compile/internal/ssa/check.go @@ -215,6 +215,9 @@ func checkFunc(f *Func) { f.Fatalf("bad FlagConstant AuxInt value for %v", v) } canHaveAuxInt = true + case auxPanicBoundsC, auxPanicBoundsCC: + canHaveAux = true + canHaveAuxInt = true default: f.Fatalf("unknown aux type for %s", v.Op) } diff --git a/src/cmd/compile/internal/ssa/op.go b/src/cmd/compile/internal/ssa/op.go index 2a3356bc5c..3bf5863360 100644 --- a/src/cmd/compile/internal/ssa/op.go +++ b/src/cmd/compile/internal/ssa/op.go @@ -6,10 +6,12 @@ package ssa import ( "cmd/compile/internal/abi" + "cmd/compile/internal/base" "cmd/compile/internal/ir" "cmd/compile/internal/types" "cmd/internal/obj" "fmt" + rtabi "internal/abi" "strings" ) @@ -365,6 +367,9 @@ const ( auxCall // aux is a *ssa.AuxCall auxCallOff // aux is a *ssa.AuxCall, AuxInt is int64 param (in+out) size + auxPanicBoundsC // constant for a bounds failure + auxPanicBoundsCC // two constants for a bounds failure + // architecture specific aux types auxARM64BitField // aux is an arm64 bitfield lsb and width packed into auxInt auxS390XRotateParams // aux is a s390x rotate parameters object encoding start bit, end bit and rotate amount @@ -523,6 +528,50 @@ func boundsABI(b int64) int { } } +// Returns the bounds error code needed by the runtime, and +// whether the x field is signed. +func (b BoundsKind) Code() (rtabi.BoundsErrorCode, bool) { + switch b { + case BoundsIndex: + return rtabi.BoundsIndex, true + case BoundsIndexU: + return rtabi.BoundsIndex, false + case BoundsSliceAlen: + return rtabi.BoundsSliceAlen, true + case BoundsSliceAlenU: + return rtabi.BoundsSliceAlen, false + case BoundsSliceAcap: + return rtabi.BoundsSliceAcap, true + case BoundsSliceAcapU: + return rtabi.BoundsSliceAcap, false + case BoundsSliceB: + return rtabi.BoundsSliceB, true + case BoundsSliceBU: + return rtabi.BoundsSliceB, false + case BoundsSlice3Alen: + return rtabi.BoundsSlice3Alen, true + case BoundsSlice3AlenU: + return rtabi.BoundsSlice3Alen, false + case BoundsSlice3Acap: + return rtabi.BoundsSlice3Acap, true + case BoundsSlice3AcapU: + return rtabi.BoundsSlice3Acap, false + case BoundsSlice3B: + return rtabi.BoundsSlice3B, true + case BoundsSlice3BU: + return rtabi.BoundsSlice3B, false + case BoundsSlice3C: + return rtabi.BoundsSlice3C, true + case BoundsSlice3CU: + return rtabi.BoundsSlice3C, false + case BoundsConvert: + return rtabi.BoundsConvert, false + default: + base.Fatalf("bad bounds kind %d", b) + return 0, false + } +} + // arm64BitField is the GO type of ARM64BitField auxInt. // if x is an ARM64BitField, then width=x&0xff, lsb=(x>>8)&0xff, and // width+lsb<64 for 64-bit variant, width+lsb<32 for 32-bit variant. diff --git a/src/cmd/compile/internal/ssa/rewrite.go b/src/cmd/compile/internal/ssa/rewrite.go index 0e7d354d6b..9c31cd9977 100644 --- a/src/cmd/compile/internal/ssa/rewrite.go +++ b/src/cmd/compile/internal/ssa/rewrite.go @@ -2673,3 +2673,32 @@ func flagify(v *Value) bool { v.AddArg(inner) return true } + +// PanicBoundsC contains a constant for a bounds failure. +type PanicBoundsC struct { + C int64 +} + +// PanicBoundsCC contains 2 constants for a bounds failure. +type PanicBoundsCC struct { + Cx int64 + Cy int64 +} + +func (p PanicBoundsC) CanBeAnSSAAux() { +} +func (p PanicBoundsCC) CanBeAnSSAAux() { +} + +func auxToPanicBoundsC(i Aux) PanicBoundsC { + return i.(PanicBoundsC) +} +func auxToPanicBoundsCC(i Aux) PanicBoundsCC { + return i.(PanicBoundsCC) +} +func panicBoundsCToAux(p PanicBoundsC) Aux { + return p +} +func panicBoundsCCToAux(p PanicBoundsCC) Aux { + return p +} diff --git a/src/cmd/compile/internal/ssagen/ssa.go b/src/cmd/compile/internal/ssagen/ssa.go index 6ca299cac5..053bfc3093 100644 --- a/src/cmd/compile/internal/ssagen/ssa.go +++ b/src/cmd/compile/internal/ssagen/ssa.go @@ -137,6 +137,8 @@ func InitConfig() { ir.Syms.Asanwrite = typecheck.LookupRuntimeFunc("asanwrite") ir.Syms.Newobject = typecheck.LookupRuntimeFunc("newobject") ir.Syms.Newproc = typecheck.LookupRuntimeFunc("newproc") + ir.Syms.PanicBounds = typecheck.LookupRuntimeFunc("panicBounds") + ir.Syms.PanicExtend = typecheck.LookupRuntimeFunc("panicExtend") ir.Syms.Panicdivide = typecheck.LookupRuntimeFunc("panicdivide") ir.Syms.PanicdottypeE = typecheck.LookupRuntimeFunc("panicdottypeE") ir.Syms.PanicdottypeI = typecheck.LookupRuntimeFunc("panicdottypeI") diff --git a/src/internal/abi/bounds.go b/src/internal/abi/bounds.go index 2266a5f3c6..d6859802d2 100644 --- a/src/internal/abi/bounds.go +++ b/src/internal/abi/bounds.go @@ -18,4 +18,96 @@ const ( BoundsSlice3B // s[?:x:y], 0 <= x <= y failed (but boundsSlice3A didn't happen) BoundsSlice3C // s[x:y:?], 0 <= x <= y failed (but boundsSlice3A/B didn't happen) BoundsConvert // (*[x]T)(s), 0 <= x <= len(s) failed + numBoundsCodes ) + +const ( + BoundsMaxReg = 15 + BoundsMaxConst = 31 +) + +// Here's how we encode PCDATA_PanicBounds entries: + +// We allow 16 registers (0-15) and 32 constants (0-31). +// Encode the following constant c: +// bits use +// ----------------------------- +// 0 x is in a register +// 1 y is in a register +// +// if x is in a register +// 2 x is signed +// [3:6] x's register number +// else +// [2:6] x's constant value +// +// if y is in a register +// [7:10] y's register number +// else +// [7:11] y's constant value +// +// The final integer is c * numBoundsCode + code + +// TODO: 32-bit + +// Encode bounds failure information into an integer for PCDATA_PanicBounds. +// Register numbers must be in 0-15. Constants must be in 0-31. +func BoundsEncode(code BoundsErrorCode, signed, xIsReg, yIsReg bool, xVal, yVal int) int { + c := int(0) + if xIsReg { + c |= 1 << 0 + if signed { + c |= 1 << 2 + } + if xVal < 0 || xVal > BoundsMaxReg { + panic("bad xReg") + } + c |= xVal << 3 + } else { + if xVal < 0 || xVal > BoundsMaxConst { + panic("bad xConst") + } + c |= xVal << 2 + } + if yIsReg { + c |= 1 << 1 + if yVal < 0 || yVal > BoundsMaxReg { + panic("bad yReg") + } + c |= yVal << 7 + } else { + if yVal < 0 || yVal > BoundsMaxConst { + panic("bad yConst") + } + c |= yVal << 7 + } + return c*int(numBoundsCodes) + int(code) +} +func BoundsDecode(v int) (code BoundsErrorCode, signed, xIsReg, yIsReg bool, xVal, yVal int) { + code = BoundsErrorCode(v % int(numBoundsCodes)) + c := v / int(numBoundsCodes) + xIsReg = c&1 != 0 + c >>= 1 + yIsReg = c&1 != 0 + c >>= 1 + if xIsReg { + signed = c&1 != 0 + c >>= 1 + xVal = c & 0xf + c >>= 4 + } else { + xVal = c & 0x1f + c >>= 5 + } + if yIsReg { + yVal = c & 0xf + c >>= 4 + } else { + yVal = c & 0x1f + c >>= 5 + } + if c != 0 { + panic("BoundsDecode decoding error") + } + return +} diff --git a/src/internal/abi/symtab.go b/src/internal/abi/symtab.go index ce322f2d75..86d6700388 100644 --- a/src/internal/abi/symtab.go +++ b/src/internal/abi/symtab.go @@ -79,6 +79,7 @@ const ( PCDATA_StackMapIndex = 1 PCDATA_InlTreeIndex = 2 PCDATA_ArgLiveIndex = 3 + PCDATA_PanicBounds = 4 FUNCDATA_ArgsPointerMaps = 0 FUNCDATA_LocalsPointerMaps = 1 diff --git a/src/runtime/panic.go b/src/runtime/panic.go index 0837b620e2..70e22836f2 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -225,6 +225,99 @@ func panicSlice3C(x int, y int) func panicSlice3CU(x uint, y int) func panicSliceConvert(x int, y int) +func panicBounds() // in asm_GOARCH.s files, called from generated code +func panicExtend() // in asm_GOARCH.s files, called from generated code (on 32-bit archs) +func panicBounds64(pc uintptr, regs *[16]int64) { // called from panicBounds on 64-bit archs + f := findfunc(pc) + v := pcdatavalue(f, abi.PCDATA_PanicBounds, pc-1) + + code, signed, xIsReg, yIsReg, xVal, yVal := abi.BoundsDecode(int(v)) + + if code == abi.BoundsIndex { + panicCheck1(pc, "index out of range") + } else { + panicCheck1(pc, "slice bounds out of range") + } + + var e boundsError + e.code = code + e.signed = signed + if xIsReg { + e.x = regs[xVal] + } else { + e.x = int64(xVal) + } + if yIsReg { + e.y = int(regs[yVal]) + } else { + e.y = yVal + } + panic(e) +} + +func panicBounds32(pc uintptr, regs *[16]int32) { // called from panicBounds on 32-bit archs + f := findfunc(pc) + v := pcdatavalue(f, abi.PCDATA_PanicBounds, pc-1) + + code, signed, xIsReg, yIsReg, xVal, yVal := abi.BoundsDecode(int(v)) + + if code == abi.BoundsIndex { + panicCheck1(pc, "index out of range") + } else { + panicCheck1(pc, "slice bounds out of range") + } + + var e boundsError + e.code = code + e.signed = signed + if xIsReg { + if signed { + e.x = int64(regs[xVal]) + } else { + e.x = int64(uint32(regs[xVal])) + } + } else { + e.x = int64(xVal) + } + if yIsReg { + e.y = int(regs[yVal]) + } else { + e.y = yVal + } + panic(e) +} + +func panicBounds32X(pc uintptr, regs *[16]int32) { // called from panicExtend on 32-bit archs + f := findfunc(pc) + v := pcdatavalue(f, abi.PCDATA_PanicBounds, pc-1) + + code, signed, xIsReg, yIsReg, xVal, yVal := abi.BoundsDecode(int(v)) + + if code == abi.BoundsIndex { + panicCheck1(pc, "index out of range") + } else { + panicCheck1(pc, "slice bounds out of range") + } + + var e boundsError + e.code = code + e.signed = signed + if xIsReg { + // Our 4-bit register numbers are actually 2 2-bit register numbers. + lo := xVal & 3 + hi := xVal >> 2 + e.x = int64(regs[hi])<<32 + int64(uint32(regs[lo])) + } else { + e.x = int64(xVal) + } + if yIsReg { + e.y = int(regs[yVal]) + } else { + e.y = yVal + } + panic(e) +} + var shiftError = error(errorString("negative shift amount")) //go:yeswritebarrierrec -- 2.51.0