From 7dccd6395cd88e69654409f46d57931d1f9b90a0 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Wed, 18 Jun 2025 15:15:29 -0700 Subject: [PATCH] cmd/compile: move arm32 over to new bounds check strategy Change-Id: I529edd805875a4833cabcf4692f0c6d4163b07d2 Reviewed-on: https://go-review.googlesource.com/c/go/+/682398 Reviewed-by: Michael Knyszek Reviewed-by: Cuong Manh Le Reviewed-by: David Chase LUCI-TryBot-Result: Go LUCI --- src/cmd/compile/internal/arm/ssa.go | 170 ++++++++++++- src/cmd/compile/internal/ssa/_gen/ARM.rules | 15 +- src/cmd/compile/internal/ssa/_gen/ARMOps.go | 25 +- src/cmd/compile/internal/ssa/opGen.go | 69 +++--- src/cmd/compile/internal/ssa/rewriteARM.go | 255 +++++++++++--------- src/runtime/asm_arm.s | 208 ++++------------ 6 files changed, 407 insertions(+), 335 deletions(-) diff --git a/src/cmd/compile/internal/arm/ssa.go b/src/cmd/compile/internal/arm/ssa.go index f129ab493d..91ef31c215 100644 --- a/src/cmd/compile/internal/arm/ssa.go +++ b/src/cmd/compile/internal/arm/ssa.go @@ -18,6 +18,7 @@ import ( "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/arm" + "internal/abi" ) // loadByType returns the load instruction of the given type. @@ -712,18 +713,167 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { p.To.Name = obj.NAME_EXTERN // AuxInt encodes how many buffer entries we need. p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1] - case ssa.OpARMLoweredPanicBoundsA, ssa.OpARMLoweredPanicBoundsB, ssa.OpARMLoweredPanicBoundsC: - p := s.Prog(obj.ACALL) - p.To.Type = obj.TYPE_MEM - p.To.Name = obj.NAME_EXTERN - p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt] - s.UseArgs(8) // space used in callee args area by assembly stubs - case ssa.OpARMLoweredPanicExtendA, ssa.OpARMLoweredPanicExtendB, ssa.OpARMLoweredPanicExtendC: - p := s.Prog(obj.ACALL) + + case ssa.OpARMLoweredPanicBoundsRR, ssa.OpARMLoweredPanicBoundsRC, ssa.OpARMLoweredPanicBoundsCR, ssa.OpARMLoweredPanicBoundsCC, + ssa.OpARMLoweredPanicExtendRR, ssa.OpARMLoweredPanicExtendRC: + // Compute the constant we put in the PCData entry for this call. + code, signed := ssa.BoundsKind(v.AuxInt).Code() + xIsReg := false + yIsReg := false + xVal := 0 + yVal := 0 + extend := false + switch v.Op { + case ssa.OpARMLoweredPanicBoundsRR: + xIsReg = true + xVal = int(v.Args[0].Reg() - arm.REG_R0) + yIsReg = true + yVal = int(v.Args[1].Reg() - arm.REG_R0) + case ssa.OpARMLoweredPanicExtendRR: + extend = true + xIsReg = true + hi := int(v.Args[0].Reg() - arm.REG_R0) + lo := int(v.Args[1].Reg() - arm.REG_R0) + xVal = hi<<2 + lo // encode 2 register numbers + yIsReg = true + yVal = int(v.Args[2].Reg() - arm.REG_R0) + case ssa.OpARMLoweredPanicBoundsRC: + xIsReg = true + xVal = int(v.Args[0].Reg() - arm.REG_R0) + c := v.Aux.(ssa.PanicBoundsC).C + if c >= 0 && c <= abi.BoundsMaxConst { + yVal = int(c) + } else { + // Move constant to a register + yIsReg = true + if yVal == xVal { + yVal = 1 + } + p := s.Prog(arm.AMOVW) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c + p.To.Type = obj.TYPE_REG + p.To.Reg = arm.REG_R0 + int16(yVal) + } + case ssa.OpARMLoweredPanicExtendRC: + extend = true + xIsReg = true + hi := int(v.Args[0].Reg() - arm.REG_R0) + lo := int(v.Args[1].Reg() - arm.REG_R0) + xVal = hi<<2 + lo // encode 2 register numbers + c := v.Aux.(ssa.PanicBoundsC).C + if c >= 0 && c <= abi.BoundsMaxConst { + yVal = int(c) + } else { + // Move constant to a register + for yVal == hi || yVal == lo { + yVal++ + } + p := s.Prog(arm.AMOVW) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c + p.To.Type = obj.TYPE_REG + p.To.Reg = arm.REG_R0 + int16(yVal) + } + case ssa.OpARMLoweredPanicBoundsCR: + yIsReg = true + yVal := int(v.Args[0].Reg() - arm.REG_R0) + c := v.Aux.(ssa.PanicBoundsC).C + if c >= 0 && c <= abi.BoundsMaxConst { + xVal = int(c) + } else if signed && int64(int32(c)) == c || !signed && int64(uint32(c)) == c { + // Move constant to a register + xIsReg = true + if xVal == yVal { + xVal = 1 + } + p := s.Prog(arm.AMOVW) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c + p.To.Type = obj.TYPE_REG + p.To.Reg = arm.REG_R0 + int16(xVal) + } else { + // Move constant to two registers + extend = true + xIsReg = true + hi := 0 + lo := 1 + if hi == yVal { + hi = 2 + } + if lo == yVal { + lo = 2 + } + xVal = hi<<2 + lo + p := s.Prog(arm.AMOVW) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c >> 32 + p.To.Type = obj.TYPE_REG + p.To.Reg = arm.REG_R0 + int16(hi) + p = s.Prog(arm.AMOVW) + p.From.Type = obj.TYPE_CONST + p.From.Offset = int64(int32(c)) + p.To.Type = obj.TYPE_REG + p.To.Reg = arm.REG_R0 + int16(lo) + } + case ssa.OpARMLoweredPanicBoundsCC: + c := v.Aux.(ssa.PanicBoundsCC).Cx + if c >= 0 && c <= abi.BoundsMaxConst { + xVal = int(c) + } else if signed && int64(int32(c)) == c || !signed && int64(uint32(c)) == c { + // Move constant to a register + xIsReg = true + p := s.Prog(arm.AMOVW) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c + p.To.Type = obj.TYPE_REG + p.To.Reg = arm.REG_R0 + int16(xVal) + } else { + // Move constant to two registers + extend = true + xIsReg = true + hi := 0 + lo := 1 + xVal = hi<<2 + lo + p := s.Prog(arm.AMOVW) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c >> 32 + p.To.Type = obj.TYPE_REG + p.To.Reg = arm.REG_R0 + int16(hi) + p = s.Prog(arm.AMOVW) + p.From.Type = obj.TYPE_CONST + p.From.Offset = int64(int32(c)) + p.To.Type = obj.TYPE_REG + p.To.Reg = arm.REG_R0 + int16(lo) + } + c = v.Aux.(ssa.PanicBoundsCC).Cy + if c >= 0 && c <= abi.BoundsMaxConst { + yVal = int(c) + } else { + // Move constant to a register + yIsReg = true + yVal = 2 + p := s.Prog(arm.AMOVW) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c + p.To.Type = obj.TYPE_REG + p.To.Reg = arm.REG_R0 + int16(yVal) + } + } + c := abi.BoundsEncode(code, signed, xIsReg, yIsReg, xVal, yVal) + + p := s.Prog(obj.APCDATA) + p.From.SetConst(abi.PCDATA_PanicBounds) + p.To.SetConst(int64(c)) + p = s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = ssagen.ExtendCheckFunc[v.AuxInt] - s.UseArgs(12) // space used in callee args area by assembly stubs + if extend { + p.To.Sym = ir.Syms.PanicExtend + } else { + p.To.Sym = ir.Syms.PanicBounds + } + case ssa.OpARMDUFFZERO: p := s.Prog(obj.ADUFFZERO) p.To.Type = obj.TYPE_MEM diff --git a/src/cmd/compile/internal/ssa/_gen/ARM.rules b/src/cmd/compile/internal/ssa/_gen/ARM.rules index 431a12f35b..18b5d6bba6 100644 --- a/src/cmd/compile/internal/ssa/_gen/ARM.rules +++ b/src/cmd/compile/internal/ssa/_gen/ARM.rules @@ -395,13 +395,16 @@ // Write barrier. (WB ...) => (LoweredWB ...) -(PanicBounds [kind] x y mem) && boundsABI(kind) == 0 => (LoweredPanicBoundsA [kind] x y mem) -(PanicBounds [kind] x y mem) && boundsABI(kind) == 1 => (LoweredPanicBoundsB [kind] x y mem) -(PanicBounds [kind] x y mem) && boundsABI(kind) == 2 => (LoweredPanicBoundsC [kind] x y mem) +(PanicBounds ...) => (LoweredPanicBoundsRR ...) +(PanicExtend ...) => (LoweredPanicExtendRR ...) -(PanicExtend [kind] hi lo y mem) && boundsABI(kind) == 0 => (LoweredPanicExtendA [kind] hi lo y mem) -(PanicExtend [kind] hi lo y mem) && boundsABI(kind) == 1 => (LoweredPanicExtendB [kind] hi lo y mem) -(PanicExtend [kind] hi lo y mem) && boundsABI(kind) == 2 => (LoweredPanicExtendC [kind] hi lo y mem) +(LoweredPanicBoundsRR [kind] x (MOVWconst [c]) mem) => (LoweredPanicBoundsRC [kind] x {PanicBoundsC{C:int64(c)}} mem) +(LoweredPanicBoundsRR [kind] (MOVWconst [c]) y mem) => (LoweredPanicBoundsCR [kind] {PanicBoundsC{C:int64(c)}} y mem) +(LoweredPanicBoundsRC [kind] {p} (MOVWconst [c]) mem) => (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:int64(c), Cy:p.C}} mem) + +(LoweredPanicExtendRR [kind] hi lo (MOVWconst [c]) mem) => (LoweredPanicExtendRC [kind] hi lo {PanicBoundsC{C:int64(c)}} mem) +(LoweredPanicExtendRR [kind] (MOVWconst [hi]) (MOVWconst [lo]) y mem) => (LoweredPanicBoundsCR [kind] {PanicBoundsC{C:int64(hi)<<32 + int64(uint32(lo))}} y mem) +(LoweredPanicExtendRC [kind] {p} (MOVWconst [hi]) (MOVWconst [lo]) mem) => (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:int64(hi)<<32+int64(uint32(lo)), Cy:p.C}} mem) // Optimizations diff --git a/src/cmd/compile/internal/ssa/_gen/ARMOps.go b/src/cmd/compile/internal/ssa/_gen/ARMOps.go index 3ad96fcac0..01cd48835e 100644 --- a/src/cmd/compile/internal/ssa/_gen/ARMOps.go +++ b/src/cmd/compile/internal/ssa/_gen/ARMOps.go @@ -94,11 +94,11 @@ func init() { gpspsbg = gpspg | buildReg("SB") fp = buildReg("F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15") callerSave = gp | fp | buildReg("g") // runtime.setg (and anything calling it) may clobber g + lr = buildReg("R14") r0 = buildReg("R0") r1 = buildReg("R1") r2 = buildReg("R2") r3 = buildReg("R3") - r4 = buildReg("R4") ) // Common regInfo var ( @@ -540,16 +540,19 @@ func init() { // See runtime/stubs.go for a more detailed discussion. {name: "LoweredGetCallerPC", reg: gp01, rematerializeable: true}, - // There are three of these functions so that they can have three different register inputs. - // When we check 0 <= c <= cap (A), then 0 <= b <= c (B), then 0 <= a <= b (C), we want the - // default registers to match so we don't need to copy registers around unnecessarily. - {name: "LoweredPanicBoundsA", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{r2, r3}}, typ: "Mem", call: true}, // arg0=idx, arg1=len, arg2=mem, returns memory. AuxInt contains report code (see PanicBounds in genericOps.go). - {name: "LoweredPanicBoundsB", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{r1, r2}}, typ: "Mem", call: true}, // arg0=idx, arg1=len, arg2=mem, returns memory. AuxInt contains report code (see PanicBounds in genericOps.go). - {name: "LoweredPanicBoundsC", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{r0, r1}}, typ: "Mem", call: true}, // arg0=idx, arg1=len, arg2=mem, returns memory. AuxInt contains report code (see PanicBounds in genericOps.go). - // Extend ops are the same as Bounds ops except the indexes are 64-bit. - {name: "LoweredPanicExtendA", argLength: 4, aux: "Int64", reg: regInfo{inputs: []regMask{r4, r2, r3}}, typ: "Mem", call: true}, // arg0=idxHi, arg1=idxLo, arg2=len, arg3=mem, returns memory. AuxInt contains report code (see PanicExtend in genericOps.go). - {name: "LoweredPanicExtendB", argLength: 4, aux: "Int64", reg: regInfo{inputs: []regMask{r4, r1, r2}}, typ: "Mem", call: true}, // arg0=idxHi, arg1=idxLo, arg2=len, arg3=mem, returns memory. AuxInt contains report code (see PanicExtend in genericOps.go). - {name: "LoweredPanicExtendC", argLength: 4, aux: "Int64", reg: regInfo{inputs: []regMask{r4, r0, r1}}, typ: "Mem", call: true}, // arg0=idxHi, arg1=idxLo, arg2=len, arg3=mem, returns memory. AuxInt contains report code (see PanicExtend in genericOps.go). + // LoweredPanicBoundsRR takes x and y, two values that caused a bounds check to fail. + // the RC and CR versions are used when one of the arguments is a constant. CC is used + // when both are constant (normally both 0, as prove derives the fact that a [0] bounds + // failure means the length must have also been 0). + // AuxInt contains a report code (see PanicBounds in genericOps.go). + {name: "LoweredPanicBoundsRR", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{gp &^ lr, gp &^ lr}}, typ: "Mem", call: true}, // arg0=x, arg1=y, arg2=mem, returns memory. + {name: "LoweredPanicBoundsRC", argLength: 2, aux: "PanicBoundsC", reg: regInfo{inputs: []regMask{gp &^ lr}}, typ: "Mem", call: true}, // arg0=x, arg1=mem, returns memory. + {name: "LoweredPanicBoundsCR", argLength: 2, aux: "PanicBoundsC", reg: regInfo{inputs: []regMask{gp &^ lr}}, typ: "Mem", call: true}, // arg0=y, arg1=mem, returns memory. + {name: "LoweredPanicBoundsCC", argLength: 1, aux: "PanicBoundsCC", reg: regInfo{}, typ: "Mem", call: true}, // arg0=mem, returns memory. + + // Same as above, but the x value is 64 bits. + {name: "LoweredPanicExtendRR", argLength: 4, aux: "Int64", reg: regInfo{inputs: []regMask{r0 | r1 | r2 | r3, r0 | r1 | r2 | r3, gp}}, typ: "Mem", call: true}, // arg0=x_hi, arg1=x_lo, arg2=y, arg3=mem, returns memory. + {name: "LoweredPanicExtendRC", argLength: 3, aux: "PanicBoundsC", reg: regInfo{inputs: []regMask{r0 | r1 | r2 | r3, r0 | r1 | r2 | r3}}, typ: "Mem", call: true}, // arg0=x_hi, arg1=x_lo, arg2=mem, returns memory. // Constant flag value. // Note: there's an "unordered" outcome for floating-point diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index 9ba4fa37c7..18c1c15a5b 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -1422,12 +1422,12 @@ const ( OpARMLoweredGetClosurePtr OpARMLoweredGetCallerSP OpARMLoweredGetCallerPC - OpARMLoweredPanicBoundsA - OpARMLoweredPanicBoundsB - OpARMLoweredPanicBoundsC - OpARMLoweredPanicExtendA - OpARMLoweredPanicExtendB - OpARMLoweredPanicExtendC + OpARMLoweredPanicBoundsRR + OpARMLoweredPanicBoundsRC + OpARMLoweredPanicBoundsCR + OpARMLoweredPanicBoundsCC + OpARMLoweredPanicExtendRR + OpARMLoweredPanicExtendRC OpARMFlagConstant OpARMInvertFlags OpARMLoweredWB @@ -19111,77 +19111,68 @@ var opcodeTable = [...]opInfo{ }, }, { - name: "LoweredPanicBoundsA", + name: "LoweredPanicBoundsRR", auxType: auxInt64, argLen: 3, call: true, reg: regInfo{ inputs: []inputInfo{ - {0, 4}, // R2 - {1, 8}, // R3 + {0, 5119}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R12 + {1, 5119}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R12 }, }, }, { - name: "LoweredPanicBoundsB", - auxType: auxInt64, - argLen: 3, + name: "LoweredPanicBoundsRC", + auxType: auxPanicBoundsC, + argLen: 2, call: true, reg: regInfo{ inputs: []inputInfo{ - {0, 2}, // R1 - {1, 4}, // R2 + {0, 5119}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R12 }, }, }, { - name: "LoweredPanicBoundsC", - auxType: auxInt64, - argLen: 3, + name: "LoweredPanicBoundsCR", + auxType: auxPanicBoundsC, + argLen: 2, call: true, reg: regInfo{ inputs: []inputInfo{ - {0, 1}, // R0 - {1, 2}, // R1 + {0, 5119}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R12 }, }, }, { - name: "LoweredPanicExtendA", - auxType: auxInt64, - argLen: 4, + name: "LoweredPanicBoundsCC", + auxType: auxPanicBoundsCC, + argLen: 1, call: true, - reg: regInfo{ - inputs: []inputInfo{ - {0, 16}, // R4 - {1, 4}, // R2 - {2, 8}, // R3 - }, - }, + reg: regInfo{}, }, { - name: "LoweredPanicExtendB", + name: "LoweredPanicExtendRR", auxType: auxInt64, argLen: 4, call: true, reg: regInfo{ inputs: []inputInfo{ - {0, 16}, // R4 - {1, 2}, // R1 - {2, 4}, // R2 + {0, 15}, // R0 R1 R2 R3 + {1, 15}, // R0 R1 R2 R3 + {2, 21503}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R12 R14 }, }, }, { - name: "LoweredPanicExtendC", - auxType: auxInt64, - argLen: 4, + name: "LoweredPanicExtendRC", + auxType: auxPanicBoundsC, + argLen: 3, call: true, reg: regInfo{ inputs: []inputInfo{ - {0, 16}, // R4 - {1, 1}, // R0 - {2, 2}, // R1 + {0, 15}, // R0 R1 R2 R3 + {1, 15}, // R0 R1 R2 R3 }, }, }, diff --git a/src/cmd/compile/internal/ssa/rewriteARM.go b/src/cmd/compile/internal/ssa/rewriteARM.go index 00517da4a1..44380cf8f5 100644 --- a/src/cmd/compile/internal/ssa/rewriteARM.go +++ b/src/cmd/compile/internal/ssa/rewriteARM.go @@ -151,6 +151,14 @@ func rewriteValueARM(v *Value) bool { return rewriteValueARM_OpARMLessThan(v) case OpARMLessThanU: return rewriteValueARM_OpARMLessThanU(v) + case OpARMLoweredPanicBoundsRC: + return rewriteValueARM_OpARMLoweredPanicBoundsRC(v) + case OpARMLoweredPanicBoundsRR: + return rewriteValueARM_OpARMLoweredPanicBoundsRR(v) + case OpARMLoweredPanicExtendRC: + return rewriteValueARM_OpARMLoweredPanicExtendRC(v) + case OpARMLoweredPanicExtendRR: + return rewriteValueARM_OpARMLoweredPanicExtendRR(v) case OpARMMOVBUload: return rewriteValueARM_OpARMMOVBUload(v) case OpARMMOVBUloadidx: @@ -745,9 +753,11 @@ func rewriteValueARM(v *Value) bool { v.Op = OpARMOR return true case OpPanicBounds: - return rewriteValueARM_OpPanicBounds(v) + v.Op = OpARMLoweredPanicBoundsRR + return true case OpPanicExtend: - return rewriteValueARM_OpPanicExtend(v) + v.Op = OpARMLoweredPanicExtendRR + return true case OpRotateLeft16: return rewriteValueARM_OpRotateLeft16(v) case OpRotateLeft32: @@ -4548,6 +4558,135 @@ func rewriteValueARM_OpARMLessThanU(v *Value) bool { } return false } +func rewriteValueARM_OpARMLoweredPanicBoundsRC(v *Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (LoweredPanicBoundsRC [kind] {p} (MOVWconst [c]) mem) + // result: (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:int64(c), Cy:p.C}} mem) + for { + kind := auxIntToInt64(v.AuxInt) + p := auxToPanicBoundsC(v.Aux) + if v_0.Op != OpARMMOVWconst { + break + } + c := auxIntToInt32(v_0.AuxInt) + mem := v_1 + v.reset(OpARMLoweredPanicBoundsCC) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCCToAux(PanicBoundsCC{Cx: int64(c), Cy: p.C}) + v.AddArg(mem) + return true + } + return false +} +func rewriteValueARM_OpARMLoweredPanicBoundsRR(v *Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (LoweredPanicBoundsRR [kind] x (MOVWconst [c]) mem) + // result: (LoweredPanicBoundsRC [kind] x {PanicBoundsC{C:int64(c)}} mem) + for { + kind := auxIntToInt64(v.AuxInt) + x := v_0 + if v_1.Op != OpARMMOVWconst { + break + } + c := auxIntToInt32(v_1.AuxInt) + mem := v_2 + v.reset(OpARMLoweredPanicBoundsRC) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCToAux(PanicBoundsC{C: int64(c)}) + v.AddArg2(x, mem) + return true + } + // match: (LoweredPanicBoundsRR [kind] (MOVWconst [c]) y mem) + // result: (LoweredPanicBoundsCR [kind] {PanicBoundsC{C:int64(c)}} y mem) + for { + kind := auxIntToInt64(v.AuxInt) + if v_0.Op != OpARMMOVWconst { + break + } + c := auxIntToInt32(v_0.AuxInt) + y := v_1 + mem := v_2 + v.reset(OpARMLoweredPanicBoundsCR) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCToAux(PanicBoundsC{C: int64(c)}) + v.AddArg2(y, mem) + return true + } + return false +} +func rewriteValueARM_OpARMLoweredPanicExtendRC(v *Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (LoweredPanicExtendRC [kind] {p} (MOVWconst [hi]) (MOVWconst [lo]) mem) + // result: (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:int64(hi)<<32+int64(uint32(lo)), Cy:p.C}} mem) + for { + kind := auxIntToInt64(v.AuxInt) + p := auxToPanicBoundsC(v.Aux) + if v_0.Op != OpARMMOVWconst { + break + } + hi := auxIntToInt32(v_0.AuxInt) + if v_1.Op != OpARMMOVWconst { + break + } + lo := auxIntToInt32(v_1.AuxInt) + mem := v_2 + v.reset(OpARMLoweredPanicBoundsCC) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCCToAux(PanicBoundsCC{Cx: int64(hi)<<32 + int64(uint32(lo)), Cy: p.C}) + v.AddArg(mem) + return true + } + return false +} +func rewriteValueARM_OpARMLoweredPanicExtendRR(v *Value) bool { + v_3 := v.Args[3] + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (LoweredPanicExtendRR [kind] hi lo (MOVWconst [c]) mem) + // result: (LoweredPanicExtendRC [kind] hi lo {PanicBoundsC{C:int64(c)}} mem) + for { + kind := auxIntToInt64(v.AuxInt) + hi := v_0 + lo := v_1 + if v_2.Op != OpARMMOVWconst { + break + } + c := auxIntToInt32(v_2.AuxInt) + mem := v_3 + v.reset(OpARMLoweredPanicExtendRC) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCToAux(PanicBoundsC{C: int64(c)}) + v.AddArg3(hi, lo, mem) + return true + } + // match: (LoweredPanicExtendRR [kind] (MOVWconst [hi]) (MOVWconst [lo]) y mem) + // result: (LoweredPanicBoundsCR [kind] {PanicBoundsC{C:int64(hi)<<32 + int64(uint32(lo))}} y mem) + for { + kind := auxIntToInt64(v.AuxInt) + if v_0.Op != OpARMMOVWconst { + break + } + hi := auxIntToInt32(v_0.AuxInt) + if v_1.Op != OpARMMOVWconst { + break + } + lo := auxIntToInt32(v_1.AuxInt) + y := v_2 + mem := v_3 + v.reset(OpARMLoweredPanicBoundsCR) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCToAux(PanicBoundsC{C: int64(hi)<<32 + int64(uint32(lo))}) + v.AddArg2(y, mem) + return true + } + return false +} func rewriteValueARM_OpARMMOVBUload(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] @@ -14969,118 +15108,6 @@ func rewriteValueARM_OpOffPtr(v *Value) bool { return true } } -func rewriteValueARM_OpPanicBounds(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (PanicBounds [kind] x y mem) - // cond: boundsABI(kind) == 0 - // result: (LoweredPanicBoundsA [kind] x y mem) - for { - kind := auxIntToInt64(v.AuxInt) - x := v_0 - y := v_1 - mem := v_2 - if !(boundsABI(kind) == 0) { - break - } - v.reset(OpARMLoweredPanicBoundsA) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg3(x, y, mem) - return true - } - // match: (PanicBounds [kind] x y mem) - // cond: boundsABI(kind) == 1 - // result: (LoweredPanicBoundsB [kind] x y mem) - for { - kind := auxIntToInt64(v.AuxInt) - x := v_0 - y := v_1 - mem := v_2 - if !(boundsABI(kind) == 1) { - break - } - v.reset(OpARMLoweredPanicBoundsB) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg3(x, y, mem) - return true - } - // match: (PanicBounds [kind] x y mem) - // cond: boundsABI(kind) == 2 - // result: (LoweredPanicBoundsC [kind] x y mem) - for { - kind := auxIntToInt64(v.AuxInt) - x := v_0 - y := v_1 - mem := v_2 - if !(boundsABI(kind) == 2) { - break - } - v.reset(OpARMLoweredPanicBoundsC) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg3(x, y, mem) - return true - } - return false -} -func rewriteValueARM_OpPanicExtend(v *Value) bool { - v_3 := v.Args[3] - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (PanicExtend [kind] hi lo y mem) - // cond: boundsABI(kind) == 0 - // result: (LoweredPanicExtendA [kind] hi lo y mem) - for { - kind := auxIntToInt64(v.AuxInt) - hi := v_0 - lo := v_1 - y := v_2 - mem := v_3 - if !(boundsABI(kind) == 0) { - break - } - v.reset(OpARMLoweredPanicExtendA) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg4(hi, lo, y, mem) - return true - } - // match: (PanicExtend [kind] hi lo y mem) - // cond: boundsABI(kind) == 1 - // result: (LoweredPanicExtendB [kind] hi lo y mem) - for { - kind := auxIntToInt64(v.AuxInt) - hi := v_0 - lo := v_1 - y := v_2 - mem := v_3 - if !(boundsABI(kind) == 1) { - break - } - v.reset(OpARMLoweredPanicExtendB) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg4(hi, lo, y, mem) - return true - } - // match: (PanicExtend [kind] hi lo y mem) - // cond: boundsABI(kind) == 2 - // result: (LoweredPanicExtendC [kind] hi lo y mem) - for { - kind := auxIntToInt64(v.AuxInt) - hi := v_0 - lo := v_1 - y := v_2 - mem := v_3 - if !(boundsABI(kind) == 2) { - break - } - v.reset(OpARMLoweredPanicExtendC) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg4(hi, lo, y, mem) - return true - } - return false -} func rewriteValueARM_OpRotateLeft16(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index ca9f0ced03..742b97f888 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -991,158 +991,56 @@ TEXT runtime·gcWriteBarrier8(SB),NOSPLIT,$0 MOVW $32, R8 JMP gcWriteBarrier<>(SB) -// Note: these functions use a special calling convention to save generated code space. -// Arguments are passed in registers, but the space for those arguments are allocated -// in the caller's stack frame. These stubs write the args into that stack space and -// then tail call to the corresponding runtime handler. -// The tail call makes these stubs disappear in backtraces. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-8 - MOVW R0, x+0(FP) - MOVW R1, y+4(FP) - JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-8 - MOVW R0, x+0(FP) - MOVW R1, y+4(FP) - JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-8 - MOVW R0, x+0(FP) - MOVW R1, y+4(FP) - JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-8 - MOVW R0, x+0(FP) - MOVW R1, y+4(FP) - JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-8 - MOVW R0, x+0(FP) - MOVW R1, y+4(FP) - JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-8 - MOVW R0, x+0(FP) - MOVW R1, y+4(FP) - JMP runtime·goPanicSlice3CU(SB) -TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSliceConvert(SB) - -// Extended versions for 64-bit indexes. -TEXT runtime·panicExtendIndex(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R0, lo+4(FP) - MOVW R1, y+8(FP) - JMP runtime·goPanicExtendIndex(SB) -TEXT runtime·panicExtendIndexU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R0, lo+4(FP) - MOVW R1, y+8(FP) - JMP runtime·goPanicExtendIndexU(SB) -TEXT runtime·panicExtendSliceAlen(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSliceAlen(SB) -TEXT runtime·panicExtendSliceAlenU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSliceAlenU(SB) -TEXT runtime·panicExtendSliceAcap(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSliceAcap(SB) -TEXT runtime·panicExtendSliceAcapU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSliceAcapU(SB) -TEXT runtime·panicExtendSliceB(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R0, lo+4(FP) - MOVW R1, y+8(FP) - JMP runtime·goPanicExtendSliceB(SB) -TEXT runtime·panicExtendSliceBU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R0, lo+4(FP) - MOVW R1, y+8(FP) - JMP runtime·goPanicExtendSliceBU(SB) -TEXT runtime·panicExtendSlice3Alen(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R2, lo+4(FP) - MOVW R3, y+8(FP) - JMP runtime·goPanicExtendSlice3Alen(SB) -TEXT runtime·panicExtendSlice3AlenU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R2, lo+4(FP) - MOVW R3, y+8(FP) - JMP runtime·goPanicExtendSlice3AlenU(SB) -TEXT runtime·panicExtendSlice3Acap(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R2, lo+4(FP) - MOVW R3, y+8(FP) - JMP runtime·goPanicExtendSlice3Acap(SB) -TEXT runtime·panicExtendSlice3AcapU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R2, lo+4(FP) - MOVW R3, y+8(FP) - JMP runtime·goPanicExtendSlice3AcapU(SB) -TEXT runtime·panicExtendSlice3B(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSlice3B(SB) -TEXT runtime·panicExtendSlice3BU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSlice3BU(SB) -TEXT runtime·panicExtendSlice3C(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R0, lo+4(FP) - MOVW R1, y+8(FP) - JMP runtime·goPanicExtendSlice3C(SB) -TEXT runtime·panicExtendSlice3CU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R0, lo+4(FP) - MOVW R1, y+8(FP) - JMP runtime·goPanicExtendSlice3CU(SB) +TEXT runtime·panicBounds(SB),NOSPLIT,$72-0 + NO_LOCAL_POINTERS + // Save all int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + MOVW R0, 12(R13) + MOVW R1, 16(R13) + MOVW R2, 20(R13) + MOVW R3, 24(R13) + MOVW R4, 28(R13) + MOVW R5, 32(R13) + MOVW R6, 36(R13) + MOVW R7, 40(R13) + MOVW R8, 44(R13) + MOVW R9, 48(R13) + // skip R10 aka G @ 52(R13) + // skip R11 aka tmp @ 56(R13) + MOVW R12, 60(R13) + // skip R13 aka SP @ 64(R13) + MOVW R14, 68(R13) + // skip R15 aka PC @ 72(R13) + + MOVW R14, 4(R13) // PC immediately after call to panicBounds + ADD $12, R13, R0 // pointer to save area + MOVW R0, 8(R13) + CALL runtime·panicBounds32(SB) + RET + +TEXT runtime·panicExtend(SB),NOSPLIT,$72-0 + NO_LOCAL_POINTERS + // Save all int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + MOVW R0, 12(R13) + MOVW R1, 16(R13) + MOVW R2, 20(R13) + MOVW R3, 24(R13) + MOVW R4, 28(R13) + MOVW R5, 32(R13) + MOVW R6, 36(R13) + MOVW R7, 40(R13) + MOVW R8, 44(R13) + MOVW R9, 48(R13) + // skip R10 aka G @ 52(R13) + // skip R11 aka tmp @ 56(R13) + MOVW R12, 60(R13) + // skip R13 aka SP @ 64(R13) + // skip R14 aka LR @ 68(R13) + // skip R15 aka PC @ 72(R13) + + MOVW R14, 4(R13) // PC immediately after call to panicExtend + ADD $12, R13, R0 // pointer to save area + MOVW R0, 8(R13) + CALL runtime·panicBounds32X(SB) + RET -- 2.51.0