From 076eae436e63f33cc5999f8e2e1822f3396af3b1 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Wed, 18 Jun 2025 15:14:00 -0700 Subject: [PATCH] cmd/compile: move amd64 and 386 over to new bounds check strategy Change-Id: I13f54f04ccb8452e625dba4249e0d56bafd1fad8 Reviewed-on: https://go-review.googlesource.com/c/go/+/682397 Reviewed-by: Michael Knyszek LUCI-TryBot-Result: Go LUCI Reviewed-by: David Chase --- src/cmd/compile/internal/amd64/ssa.go | 88 +++++- src/cmd/compile/internal/ssa/_gen/386.rules | 15 +- src/cmd/compile/internal/ssa/_gen/386Ops.go | 24 +- src/cmd/compile/internal/ssa/_gen/AMD64.rules | 8 +- src/cmd/compile/internal/ssa/_gen/AMD64Ops.go | 16 +- src/cmd/compile/internal/ssa/opGen.go | 107 ++++---- src/cmd/compile/internal/ssa/rewrite386.go | 255 ++++++++++-------- src/cmd/compile/internal/ssa/rewriteAMD64.go | 143 ++++++---- src/cmd/compile/internal/x86/ssa.go | 167 +++++++++++- src/cmd/internal/obj/x86/obj6.go | 13 +- src/runtime/asm_386.s | 196 +++----------- src/runtime/asm_amd64.s | 89 ++---- 12 files changed, 631 insertions(+), 490 deletions(-) diff --git a/src/cmd/compile/internal/amd64/ssa.go b/src/cmd/compile/internal/amd64/ssa.go index 3af513773d..9fce5cfc31 100644 --- a/src/cmd/compile/internal/amd64/ssa.go +++ b/src/cmd/compile/internal/amd64/ssa.go @@ -17,6 +17,7 @@ import ( "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/x86" + "internal/abi" ) // ssaMarkMoves marks any MOVXconst ops that need to avoid clobbering flags. @@ -1135,12 +1136,91 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { // AuxInt encodes how many buffer entries we need. p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1] - case ssa.OpAMD64LoweredPanicBoundsA, ssa.OpAMD64LoweredPanicBoundsB, ssa.OpAMD64LoweredPanicBoundsC: - p := s.Prog(obj.ACALL) + case ssa.OpAMD64LoweredPanicBoundsRR, ssa.OpAMD64LoweredPanicBoundsRC, ssa.OpAMD64LoweredPanicBoundsCR, ssa.OpAMD64LoweredPanicBoundsCC: + // Compute the constant we put in the PCData entry for this call. + code, signed := ssa.BoundsKind(v.AuxInt).Code() + xIsReg := false + yIsReg := false + xVal := 0 + yVal := 0 + switch v.Op { + case ssa.OpAMD64LoweredPanicBoundsRR: + xIsReg = true + xVal = int(v.Args[0].Reg() - x86.REG_AX) + yIsReg = true + yVal = int(v.Args[1].Reg() - x86.REG_AX) + case ssa.OpAMD64LoweredPanicBoundsRC: + xIsReg = true + xVal = int(v.Args[0].Reg() - x86.REG_AX) + c := v.Aux.(ssa.PanicBoundsC).C + if c >= 0 && c <= abi.BoundsMaxConst { + yVal = int(c) + } else { + // Move constant to a register + yIsReg = true + if yVal == xVal { + yVal = 1 + } + p := s.Prog(x86.AMOVQ) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c + p.To.Type = obj.TYPE_REG + p.To.Reg = x86.REG_AX + int16(yVal) + } + case ssa.OpAMD64LoweredPanicBoundsCR: + yIsReg = true + yVal := int(v.Args[0].Reg() - x86.REG_AX) + c := v.Aux.(ssa.PanicBoundsC).C + if c >= 0 && c <= abi.BoundsMaxConst { + xVal = int(c) + } else { + // Move constant to a register + xIsReg = true + if xVal == yVal { + xVal = 1 + } + p := s.Prog(x86.AMOVQ) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c + p.To.Type = obj.TYPE_REG + p.To.Reg = x86.REG_AX + int16(xVal) + } + case ssa.OpAMD64LoweredPanicBoundsCC: + c := v.Aux.(ssa.PanicBoundsCC).Cx + if c >= 0 && c <= abi.BoundsMaxConst { + xVal = int(c) + } else { + // Move constant to a register + xIsReg = true + p := s.Prog(x86.AMOVQ) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c + p.To.Type = obj.TYPE_REG + p.To.Reg = x86.REG_AX + int16(xVal) + } + c = v.Aux.(ssa.PanicBoundsCC).Cy + if c >= 0 && c <= abi.BoundsMaxConst { + yVal = int(c) + } else { + // Move constant to a register + yIsReg = true + yVal = 1 + p := s.Prog(x86.AMOVQ) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c + p.To.Type = obj.TYPE_REG + p.To.Reg = x86.REG_AX + int16(yVal) + } + } + c := abi.BoundsEncode(code, signed, xIsReg, yIsReg, xVal, yVal) + + p := s.Prog(obj.APCDATA) + p.From.SetConst(abi.PCDATA_PanicBounds) + p.To.SetConst(int64(c)) + p = s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt] - s.UseArgs(int64(2 * types.PtrSize)) // space used in callee args area by assembly stubs + p.To.Sym = ir.Syms.PanicBounds case ssa.OpAMD64NEGQ, ssa.OpAMD64NEGL, ssa.OpAMD64BSWAPQ, ssa.OpAMD64BSWAPL, diff --git a/src/cmd/compile/internal/ssa/_gen/386.rules b/src/cmd/compile/internal/ssa/_gen/386.rules index 97414913e4..5f11502419 100644 --- a/src/cmd/compile/internal/ssa/_gen/386.rules +++ b/src/cmd/compile/internal/ssa/_gen/386.rules @@ -363,13 +363,16 @@ // Write barrier. (WB ...) => (LoweredWB ...) -(PanicBounds [kind] x y mem) && boundsABI(kind) == 0 => (LoweredPanicBoundsA [kind] x y mem) -(PanicBounds [kind] x y mem) && boundsABI(kind) == 1 => (LoweredPanicBoundsB [kind] x y mem) -(PanicBounds [kind] x y mem) && boundsABI(kind) == 2 => (LoweredPanicBoundsC [kind] x y mem) +(PanicBounds ...) => (LoweredPanicBoundsRR ...) +(PanicExtend ...) => (LoweredPanicExtendRR ...) -(PanicExtend [kind] hi lo y mem) && boundsABI(kind) == 0 => (LoweredPanicExtendA [kind] hi lo y mem) -(PanicExtend [kind] hi lo y mem) && boundsABI(kind) == 1 => (LoweredPanicExtendB [kind] hi lo y mem) -(PanicExtend [kind] hi lo y mem) && boundsABI(kind) == 2 => (LoweredPanicExtendC [kind] hi lo y mem) +(LoweredPanicBoundsRR [kind] x (MOVLconst [c]) mem) => (LoweredPanicBoundsRC [kind] x {PanicBoundsC{C:int64(c)}} mem) +(LoweredPanicBoundsRR [kind] (MOVLconst [c]) y mem) => (LoweredPanicBoundsCR [kind] {PanicBoundsC{C:int64(c)}} y mem) +(LoweredPanicBoundsRC [kind] {p} (MOVLconst [c]) mem) => (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:int64(c), Cy:p.C}} mem) + +(LoweredPanicExtendRR [kind] hi lo (MOVLconst [c]) mem) => (LoweredPanicExtendRC [kind] hi lo {PanicBoundsC{C:int64(c)}} mem) +(LoweredPanicExtendRR [kind] (MOVLconst [hi]) (MOVLconst [lo]) y mem) => (LoweredPanicBoundsCR [kind] {PanicBoundsC{C:int64(hi)<<32 + int64(uint32(lo))}} y mem) +(LoweredPanicExtendRC [kind] {p} (MOVLconst [hi]) (MOVLconst [lo]) mem) => (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:int64(hi)<<32+int64(uint32(lo)), Cy:p.C}} mem) // *************************** // Above: lowering rules diff --git a/src/cmd/compile/internal/ssa/_gen/386Ops.go b/src/cmd/compile/internal/ssa/_gen/386Ops.go index a976a91fb8..60599a33ab 100644 --- a/src/cmd/compile/internal/ssa/_gen/386Ops.go +++ b/src/cmd/compile/internal/ssa/_gen/386Ops.go @@ -76,7 +76,6 @@ func init() { cx = buildReg("CX") dx = buildReg("DX") bx = buildReg("BX") - si = buildReg("SI") gp = buildReg("AX CX DX BX BP SI DI") fp = buildReg("X0 X1 X2 X3 X4 X5 X6 X7") gpsp = gp | buildReg("SP") @@ -523,16 +522,19 @@ func init() { // Returns a pointer to a write barrier buffer in DI. {name: "LoweredWB", argLength: 1, reg: regInfo{clobbers: callerSave &^ gp, outputs: []regMask{buildReg("DI")}}, clobberFlags: true, aux: "Int64"}, - // There are three of these functions so that they can have three different register inputs. - // When we check 0 <= c <= cap (A), then 0 <= b <= c (B), then 0 <= a <= b (C), we want the - // default registers to match so we don't need to copy registers around unnecessarily. - {name: "LoweredPanicBoundsA", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{dx, bx}}, typ: "Mem", call: true}, // arg0=idx, arg1=len, arg2=mem, returns memory. AuxInt contains report code (see PanicBounds in genericOps.go). - {name: "LoweredPanicBoundsB", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{cx, dx}}, typ: "Mem", call: true}, // arg0=idx, arg1=len, arg2=mem, returns memory. AuxInt contains report code (see PanicBounds in genericOps.go). - {name: "LoweredPanicBoundsC", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{ax, cx}}, typ: "Mem", call: true}, // arg0=idx, arg1=len, arg2=mem, returns memory. AuxInt contains report code (see PanicBounds in genericOps.go). - // Extend ops are the same as Bounds ops except the indexes are 64-bit. - {name: "LoweredPanicExtendA", argLength: 4, aux: "Int64", reg: regInfo{inputs: []regMask{si, dx, bx}}, typ: "Mem", call: true}, // arg0=idxHi, arg1=idxLo, arg2=len, arg3=mem, returns memory. AuxInt contains report code (see PanicExtend in genericOps.go). - {name: "LoweredPanicExtendB", argLength: 4, aux: "Int64", reg: regInfo{inputs: []regMask{si, cx, dx}}, typ: "Mem", call: true}, // arg0=idxHi, arg1=idxLo, arg2=len, arg3=mem, returns memory. AuxInt contains report code (see PanicExtend in genericOps.go). - {name: "LoweredPanicExtendC", argLength: 4, aux: "Int64", reg: regInfo{inputs: []regMask{si, ax, cx}}, typ: "Mem", call: true}, // arg0=idxHi, arg1=idxLo, arg2=len, arg3=mem, returns memory. AuxInt contains report code (see PanicExtend in genericOps.go). + // LoweredPanicBoundsRR takes x and y, two values that caused a bounds check to fail. + // the RC and CR versions are used when one of the arguments is a constant. CC is used + // when both are constant (normally both 0, as prove derives the fact that a [0] bounds + // failure means the length must have also been 0). + // AuxInt contains a report code (see PanicBounds in genericOps.go). + {name: "LoweredPanicBoundsRR", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{gp, gp}}, typ: "Mem", call: true}, // arg0=x, arg1=y, arg2=mem, returns memory. + {name: "LoweredPanicBoundsRC", argLength: 2, aux: "PanicBoundsC", reg: regInfo{inputs: []regMask{gp}}, typ: "Mem", call: true}, // arg0=x, arg1=mem, returns memory. + {name: "LoweredPanicBoundsCR", argLength: 2, aux: "PanicBoundsC", reg: regInfo{inputs: []regMask{gp}}, typ: "Mem", call: true}, // arg0=y, arg1=mem, returns memory. + {name: "LoweredPanicBoundsCC", argLength: 1, aux: "PanicBoundsCC", reg: regInfo{}, typ: "Mem", call: true}, // arg0=mem, returns memory. + + // Same as above, but the x value is 64 bits. + {name: "LoweredPanicExtendRR", argLength: 4, aux: "Int64", reg: regInfo{inputs: []regMask{ax | cx | dx | bx, ax | cx | dx | bx, gp}}, typ: "Mem", call: true}, // arg0=x_hi, arg1=x_lo, arg2=y, arg3=mem, returns memory. + {name: "LoweredPanicExtendRC", argLength: 3, aux: "PanicBoundsC", reg: regInfo{inputs: []regMask{ax | cx | dx | bx, ax | cx | dx | bx}}, typ: "Mem", call: true}, // arg0=x_hi, arg1=x_lo, arg2=mem, returns memory. // Constant flag values. For any comparison, there are 5 possible // outcomes: the three from the signed total order (<,==,>) and the diff --git a/src/cmd/compile/internal/ssa/_gen/AMD64.rules b/src/cmd/compile/internal/ssa/_gen/AMD64.rules index d55dfe70ac..6013e81115 100644 --- a/src/cmd/compile/internal/ssa/_gen/AMD64.rules +++ b/src/cmd/compile/internal/ssa/_gen/AMD64.rules @@ -558,9 +558,11 @@ // Write barrier. (WB ...) => (LoweredWB ...) -(PanicBounds [kind] x y mem) && boundsABI(kind) == 0 => (LoweredPanicBoundsA [kind] x y mem) -(PanicBounds [kind] x y mem) && boundsABI(kind) == 1 => (LoweredPanicBoundsB [kind] x y mem) -(PanicBounds [kind] x y mem) && boundsABI(kind) == 2 => (LoweredPanicBoundsC [kind] x y mem) +(PanicBounds ...) => (LoweredPanicBoundsRR ...) +(LoweredPanicBoundsRR [kind] x (MOVQconst [c]) mem) => (LoweredPanicBoundsRC [kind] x {PanicBoundsC{C:c}} mem) +(LoweredPanicBoundsRR [kind] (MOVQconst [c]) y mem) => (LoweredPanicBoundsCR [kind] {PanicBoundsC{C:c}} y mem) +(LoweredPanicBoundsRC [kind] {p} (MOVQconst [c]) mem) => (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:c, Cy:p.C}} mem) +(LoweredPanicBoundsCR [kind] {p} (MOVQconst [c]) mem) => (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:p.C, Cy:c}} mem) // lowering rotates (RotateLeft8 ...) => (ROLB ...) diff --git a/src/cmd/compile/internal/ssa/_gen/AMD64Ops.go b/src/cmd/compile/internal/ssa/_gen/AMD64Ops.go index 0f17843565..dc29559b04 100644 --- a/src/cmd/compile/internal/ssa/_gen/AMD64Ops.go +++ b/src/cmd/compile/internal/ssa/_gen/AMD64Ops.go @@ -95,7 +95,6 @@ func init() { ax = buildReg("AX") cx = buildReg("CX") dx = buildReg("DX") - bx = buildReg("BX") gp = buildReg("AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R15") g = buildReg("g") fp = buildReg("X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14") @@ -986,12 +985,15 @@ func init() { {name: "LoweredHasCPUFeature", argLength: 0, reg: gp01, rematerializeable: true, typ: "UInt64", aux: "Sym", symEffect: "None"}, - // There are three of these functions so that they can have three different register inputs. - // When we check 0 <= c <= cap (A), then 0 <= b <= c (B), then 0 <= a <= b (C), we want the - // default registers to match so we don't need to copy registers around unnecessarily. - {name: "LoweredPanicBoundsA", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{dx, bx}}, typ: "Mem", call: true}, // arg0=idx, arg1=len, arg2=mem, returns memory. AuxInt contains report code (see PanicBounds in generic.go). - {name: "LoweredPanicBoundsB", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{cx, dx}}, typ: "Mem", call: true}, // arg0=idx, arg1=len, arg2=mem, returns memory. AuxInt contains report code (see PanicBounds in generic.go). - {name: "LoweredPanicBoundsC", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{ax, cx}}, typ: "Mem", call: true}, // arg0=idx, arg1=len, arg2=mem, returns memory. AuxInt contains report code (see PanicBounds in generic.go). + // LoweredPanicBoundsRR takes x and y, two values that caused a bounds check to fail. + // the RC and CR versions are used when one of the arguments is a constant. CC is used + // when both are constant (normally both 0, as prove derives the fact that a [0] bounds + // failure means the length must have also been 0). + // AuxInt contains a report code (see PanicBounds in genericOps.go). + {name: "LoweredPanicBoundsRR", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{gp, gp}}, typ: "Mem", call: true}, // arg0=x, arg1=y, arg2=mem, returns memory. + {name: "LoweredPanicBoundsRC", argLength: 2, aux: "PanicBoundsC", reg: regInfo{inputs: []regMask{gp}}, typ: "Mem", call: true}, // arg0=x, arg1=mem, returns memory. + {name: "LoweredPanicBoundsCR", argLength: 2, aux: "PanicBoundsC", reg: regInfo{inputs: []regMask{gp}}, typ: "Mem", call: true}, // arg0=y, arg1=mem, returns memory. + {name: "LoweredPanicBoundsCC", argLength: 1, aux: "PanicBoundsCC", reg: regInfo{}, typ: "Mem", call: true}, // arg0=mem, returns memory. // Constant flag values. For any comparison, there are 5 possible // outcomes: the three from the signed total order (<,==,>) and the diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index a6c0212557..9ba4fa37c7 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -569,12 +569,12 @@ const ( Op386LoweredGetCallerSP Op386LoweredNilCheck Op386LoweredWB - Op386LoweredPanicBoundsA - Op386LoweredPanicBoundsB - Op386LoweredPanicBoundsC - Op386LoweredPanicExtendA - Op386LoweredPanicExtendB - Op386LoweredPanicExtendC + Op386LoweredPanicBoundsRR + Op386LoweredPanicBoundsRC + Op386LoweredPanicBoundsCR + Op386LoweredPanicBoundsCC + Op386LoweredPanicExtendRR + Op386LoweredPanicExtendRC Op386FlagEQ Op386FlagLT_ULT Op386FlagLT_UGT @@ -1067,9 +1067,10 @@ const ( OpAMD64LoweredNilCheck OpAMD64LoweredWB OpAMD64LoweredHasCPUFeature - OpAMD64LoweredPanicBoundsA - OpAMD64LoweredPanicBoundsB - OpAMD64LoweredPanicBoundsC + OpAMD64LoweredPanicBoundsRR + OpAMD64LoweredPanicBoundsRC + OpAMD64LoweredPanicBoundsCR + OpAMD64LoweredPanicBoundsCC OpAMD64FlagEQ OpAMD64FlagLT_ULT OpAMD64FlagLT_UGT @@ -6578,77 +6579,68 @@ var opcodeTable = [...]opInfo{ }, }, { - name: "LoweredPanicBoundsA", + name: "LoweredPanicBoundsRR", auxType: auxInt64, argLen: 3, call: true, reg: regInfo{ inputs: []inputInfo{ - {0, 4}, // DX - {1, 8}, // BX + {0, 239}, // AX CX DX BX BP SI DI + {1, 239}, // AX CX DX BX BP SI DI }, }, }, { - name: "LoweredPanicBoundsB", - auxType: auxInt64, - argLen: 3, + name: "LoweredPanicBoundsRC", + auxType: auxPanicBoundsC, + argLen: 2, call: true, reg: regInfo{ inputs: []inputInfo{ - {0, 2}, // CX - {1, 4}, // DX + {0, 239}, // AX CX DX BX BP SI DI }, }, }, { - name: "LoweredPanicBoundsC", - auxType: auxInt64, - argLen: 3, + name: "LoweredPanicBoundsCR", + auxType: auxPanicBoundsC, + argLen: 2, call: true, reg: regInfo{ inputs: []inputInfo{ - {0, 1}, // AX - {1, 2}, // CX + {0, 239}, // AX CX DX BX BP SI DI }, }, }, { - name: "LoweredPanicExtendA", - auxType: auxInt64, - argLen: 4, + name: "LoweredPanicBoundsCC", + auxType: auxPanicBoundsCC, + argLen: 1, call: true, - reg: regInfo{ - inputs: []inputInfo{ - {0, 64}, // SI - {1, 4}, // DX - {2, 8}, // BX - }, - }, + reg: regInfo{}, }, { - name: "LoweredPanicExtendB", + name: "LoweredPanicExtendRR", auxType: auxInt64, argLen: 4, call: true, reg: regInfo{ inputs: []inputInfo{ - {0, 64}, // SI - {1, 2}, // CX - {2, 4}, // DX + {0, 15}, // AX CX DX BX + {1, 15}, // AX CX DX BX + {2, 239}, // AX CX DX BX BP SI DI }, }, }, { - name: "LoweredPanicExtendC", - auxType: auxInt64, - argLen: 4, + name: "LoweredPanicExtendRC", + auxType: auxPanicBoundsC, + argLen: 3, call: true, reg: regInfo{ inputs: []inputInfo{ - {0, 64}, // SI - {1, 1}, // AX - {2, 2}, // CX + {0, 15}, // AX CX DX BX + {1, 15}, // AX CX DX BX }, }, }, @@ -14056,41 +14048,46 @@ var opcodeTable = [...]opInfo{ }, }, { - name: "LoweredPanicBoundsA", + name: "LoweredPanicBoundsRR", auxType: auxInt64, argLen: 3, call: true, reg: regInfo{ inputs: []inputInfo{ - {0, 4}, // DX - {1, 8}, // BX + {0, 49135}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R15 + {1, 49135}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R15 }, }, }, { - name: "LoweredPanicBoundsB", - auxType: auxInt64, - argLen: 3, + name: "LoweredPanicBoundsRC", + auxType: auxPanicBoundsC, + argLen: 2, call: true, reg: regInfo{ inputs: []inputInfo{ - {0, 2}, // CX - {1, 4}, // DX + {0, 49135}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R15 }, }, }, { - name: "LoweredPanicBoundsC", - auxType: auxInt64, - argLen: 3, + name: "LoweredPanicBoundsCR", + auxType: auxPanicBoundsC, + argLen: 2, call: true, reg: regInfo{ inputs: []inputInfo{ - {0, 1}, // AX - {1, 2}, // CX + {0, 49135}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R15 }, }, }, + { + name: "LoweredPanicBoundsCC", + auxType: auxPanicBoundsCC, + argLen: 1, + call: true, + reg: regInfo{}, + }, { name: "FlagEQ", argLen: 0, diff --git a/src/cmd/compile/internal/ssa/rewrite386.go b/src/cmd/compile/internal/ssa/rewrite386.go index 9ece0e4eb7..0495438710 100644 --- a/src/cmd/compile/internal/ssa/rewrite386.go +++ b/src/cmd/compile/internal/ssa/rewrite386.go @@ -75,6 +75,14 @@ func rewriteValue386(v *Value) bool { return rewriteValue386_Op386LEAL4(v) case Op386LEAL8: return rewriteValue386_Op386LEAL8(v) + case Op386LoweredPanicBoundsRC: + return rewriteValue386_Op386LoweredPanicBoundsRC(v) + case Op386LoweredPanicBoundsRR: + return rewriteValue386_Op386LoweredPanicBoundsRR(v) + case Op386LoweredPanicExtendRC: + return rewriteValue386_Op386LoweredPanicExtendRC(v) + case Op386LoweredPanicExtendRR: + return rewriteValue386_Op386LoweredPanicExtendRR(v) case Op386MOVBLSX: return rewriteValue386_Op386MOVBLSX(v) case Op386MOVBLSXload: @@ -558,9 +566,11 @@ func rewriteValue386(v *Value) bool { v.Op = Op386ORL return true case OpPanicBounds: - return rewriteValue386_OpPanicBounds(v) + v.Op = Op386LoweredPanicBoundsRR + return true case OpPanicExtend: - return rewriteValue386_OpPanicExtend(v) + v.Op = Op386LoweredPanicExtendRR + return true case OpRotateLeft16: v.Op = Op386ROLW return true @@ -3398,6 +3408,135 @@ func rewriteValue386_Op386LEAL8(v *Value) bool { } return false } +func rewriteValue386_Op386LoweredPanicBoundsRC(v *Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (LoweredPanicBoundsRC [kind] {p} (MOVLconst [c]) mem) + // result: (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:int64(c), Cy:p.C}} mem) + for { + kind := auxIntToInt64(v.AuxInt) + p := auxToPanicBoundsC(v.Aux) + if v_0.Op != Op386MOVLconst { + break + } + c := auxIntToInt32(v_0.AuxInt) + mem := v_1 + v.reset(Op386LoweredPanicBoundsCC) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCCToAux(PanicBoundsCC{Cx: int64(c), Cy: p.C}) + v.AddArg(mem) + return true + } + return false +} +func rewriteValue386_Op386LoweredPanicBoundsRR(v *Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (LoweredPanicBoundsRR [kind] x (MOVLconst [c]) mem) + // result: (LoweredPanicBoundsRC [kind] x {PanicBoundsC{C:int64(c)}} mem) + for { + kind := auxIntToInt64(v.AuxInt) + x := v_0 + if v_1.Op != Op386MOVLconst { + break + } + c := auxIntToInt32(v_1.AuxInt) + mem := v_2 + v.reset(Op386LoweredPanicBoundsRC) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCToAux(PanicBoundsC{C: int64(c)}) + v.AddArg2(x, mem) + return true + } + // match: (LoweredPanicBoundsRR [kind] (MOVLconst [c]) y mem) + // result: (LoweredPanicBoundsCR [kind] {PanicBoundsC{C:int64(c)}} y mem) + for { + kind := auxIntToInt64(v.AuxInt) + if v_0.Op != Op386MOVLconst { + break + } + c := auxIntToInt32(v_0.AuxInt) + y := v_1 + mem := v_2 + v.reset(Op386LoweredPanicBoundsCR) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCToAux(PanicBoundsC{C: int64(c)}) + v.AddArg2(y, mem) + return true + } + return false +} +func rewriteValue386_Op386LoweredPanicExtendRC(v *Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (LoweredPanicExtendRC [kind] {p} (MOVLconst [hi]) (MOVLconst [lo]) mem) + // result: (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:int64(hi)<<32+int64(uint32(lo)), Cy:p.C}} mem) + for { + kind := auxIntToInt64(v.AuxInt) + p := auxToPanicBoundsC(v.Aux) + if v_0.Op != Op386MOVLconst { + break + } + hi := auxIntToInt32(v_0.AuxInt) + if v_1.Op != Op386MOVLconst { + break + } + lo := auxIntToInt32(v_1.AuxInt) + mem := v_2 + v.reset(Op386LoweredPanicBoundsCC) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCCToAux(PanicBoundsCC{Cx: int64(hi)<<32 + int64(uint32(lo)), Cy: p.C}) + v.AddArg(mem) + return true + } + return false +} +func rewriteValue386_Op386LoweredPanicExtendRR(v *Value) bool { + v_3 := v.Args[3] + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (LoweredPanicExtendRR [kind] hi lo (MOVLconst [c]) mem) + // result: (LoweredPanicExtendRC [kind] hi lo {PanicBoundsC{C:int64(c)}} mem) + for { + kind := auxIntToInt64(v.AuxInt) + hi := v_0 + lo := v_1 + if v_2.Op != Op386MOVLconst { + break + } + c := auxIntToInt32(v_2.AuxInt) + mem := v_3 + v.reset(Op386LoweredPanicExtendRC) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCToAux(PanicBoundsC{C: int64(c)}) + v.AddArg3(hi, lo, mem) + return true + } + // match: (LoweredPanicExtendRR [kind] (MOVLconst [hi]) (MOVLconst [lo]) y mem) + // result: (LoweredPanicBoundsCR [kind] {PanicBoundsC{C:int64(hi)<<32 + int64(uint32(lo))}} y mem) + for { + kind := auxIntToInt64(v.AuxInt) + if v_0.Op != Op386MOVLconst { + break + } + hi := auxIntToInt32(v_0.AuxInt) + if v_1.Op != Op386MOVLconst { + break + } + lo := auxIntToInt32(v_1.AuxInt) + y := v_2 + mem := v_3 + v.reset(Op386LoweredPanicBoundsCR) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCToAux(PanicBoundsC{C: int64(hi)<<32 + int64(uint32(lo))}) + v.AddArg2(y, mem) + return true + } + return false +} func rewriteValue386_Op386MOVBLSX(v *Value) bool { v_0 := v.Args[0] b := v.Block @@ -9313,118 +9452,6 @@ func rewriteValue386_OpOffPtr(v *Value) bool { return true } } -func rewriteValue386_OpPanicBounds(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (PanicBounds [kind] x y mem) - // cond: boundsABI(kind) == 0 - // result: (LoweredPanicBoundsA [kind] x y mem) - for { - kind := auxIntToInt64(v.AuxInt) - x := v_0 - y := v_1 - mem := v_2 - if !(boundsABI(kind) == 0) { - break - } - v.reset(Op386LoweredPanicBoundsA) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg3(x, y, mem) - return true - } - // match: (PanicBounds [kind] x y mem) - // cond: boundsABI(kind) == 1 - // result: (LoweredPanicBoundsB [kind] x y mem) - for { - kind := auxIntToInt64(v.AuxInt) - x := v_0 - y := v_1 - mem := v_2 - if !(boundsABI(kind) == 1) { - break - } - v.reset(Op386LoweredPanicBoundsB) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg3(x, y, mem) - return true - } - // match: (PanicBounds [kind] x y mem) - // cond: boundsABI(kind) == 2 - // result: (LoweredPanicBoundsC [kind] x y mem) - for { - kind := auxIntToInt64(v.AuxInt) - x := v_0 - y := v_1 - mem := v_2 - if !(boundsABI(kind) == 2) { - break - } - v.reset(Op386LoweredPanicBoundsC) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg3(x, y, mem) - return true - } - return false -} -func rewriteValue386_OpPanicExtend(v *Value) bool { - v_3 := v.Args[3] - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (PanicExtend [kind] hi lo y mem) - // cond: boundsABI(kind) == 0 - // result: (LoweredPanicExtendA [kind] hi lo y mem) - for { - kind := auxIntToInt64(v.AuxInt) - hi := v_0 - lo := v_1 - y := v_2 - mem := v_3 - if !(boundsABI(kind) == 0) { - break - } - v.reset(Op386LoweredPanicExtendA) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg4(hi, lo, y, mem) - return true - } - // match: (PanicExtend [kind] hi lo y mem) - // cond: boundsABI(kind) == 1 - // result: (LoweredPanicExtendB [kind] hi lo y mem) - for { - kind := auxIntToInt64(v.AuxInt) - hi := v_0 - lo := v_1 - y := v_2 - mem := v_3 - if !(boundsABI(kind) == 1) { - break - } - v.reset(Op386LoweredPanicExtendB) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg4(hi, lo, y, mem) - return true - } - // match: (PanicExtend [kind] hi lo y mem) - // cond: boundsABI(kind) == 2 - // result: (LoweredPanicExtendC [kind] hi lo y mem) - for { - kind := auxIntToInt64(v.AuxInt) - hi := v_0 - lo := v_1 - y := v_2 - mem := v_3 - if !(boundsABI(kind) == 2) { - break - } - v.reset(Op386LoweredPanicExtendC) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg4(hi, lo, y, mem) - return true - } - return false -} func rewriteValue386_OpRsh16Ux16(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] diff --git a/src/cmd/compile/internal/ssa/rewriteAMD64.go b/src/cmd/compile/internal/ssa/rewriteAMD64.go index 3d7af5f365..d2c136369e 100644 --- a/src/cmd/compile/internal/ssa/rewriteAMD64.go +++ b/src/cmd/compile/internal/ssa/rewriteAMD64.go @@ -215,6 +215,12 @@ func rewriteValueAMD64(v *Value) bool { return rewriteValueAMD64_OpAMD64LEAQ4(v) case OpAMD64LEAQ8: return rewriteValueAMD64_OpAMD64LEAQ8(v) + case OpAMD64LoweredPanicBoundsCR: + return rewriteValueAMD64_OpAMD64LoweredPanicBoundsCR(v) + case OpAMD64LoweredPanicBoundsRC: + return rewriteValueAMD64_OpAMD64LoweredPanicBoundsRC(v) + case OpAMD64LoweredPanicBoundsRR: + return rewriteValueAMD64_OpAMD64LoweredPanicBoundsRR(v) case OpAMD64MOVBELstore: return rewriteValueAMD64_OpAMD64MOVBELstore(v) case OpAMD64MOVBEQstore: @@ -977,7 +983,8 @@ func rewriteValueAMD64(v *Value) bool { v.Op = OpAMD64ORL return true case OpPanicBounds: - return rewriteValueAMD64_OpPanicBounds(v) + v.Op = OpAMD64LoweredPanicBoundsRR + return true case OpPopCount16: return rewriteValueAMD64_OpPopCount16(v) case OpPopCount32: @@ -9637,6 +9644,86 @@ func rewriteValueAMD64_OpAMD64LEAQ8(v *Value) bool { } return false } +func rewriteValueAMD64_OpAMD64LoweredPanicBoundsCR(v *Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (LoweredPanicBoundsCR [kind] {p} (MOVQconst [c]) mem) + // result: (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:p.C, Cy:c}} mem) + for { + kind := auxIntToInt64(v.AuxInt) + p := auxToPanicBoundsC(v.Aux) + if v_0.Op != OpAMD64MOVQconst { + break + } + c := auxIntToInt64(v_0.AuxInt) + mem := v_1 + v.reset(OpAMD64LoweredPanicBoundsCC) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCCToAux(PanicBoundsCC{Cx: p.C, Cy: c}) + v.AddArg(mem) + return true + } + return false +} +func rewriteValueAMD64_OpAMD64LoweredPanicBoundsRC(v *Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (LoweredPanicBoundsRC [kind] {p} (MOVQconst [c]) mem) + // result: (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:c, Cy:p.C}} mem) + for { + kind := auxIntToInt64(v.AuxInt) + p := auxToPanicBoundsC(v.Aux) + if v_0.Op != OpAMD64MOVQconst { + break + } + c := auxIntToInt64(v_0.AuxInt) + mem := v_1 + v.reset(OpAMD64LoweredPanicBoundsCC) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCCToAux(PanicBoundsCC{Cx: c, Cy: p.C}) + v.AddArg(mem) + return true + } + return false +} +func rewriteValueAMD64_OpAMD64LoweredPanicBoundsRR(v *Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (LoweredPanicBoundsRR [kind] x (MOVQconst [c]) mem) + // result: (LoweredPanicBoundsRC [kind] x {PanicBoundsC{C:c}} mem) + for { + kind := auxIntToInt64(v.AuxInt) + x := v_0 + if v_1.Op != OpAMD64MOVQconst { + break + } + c := auxIntToInt64(v_1.AuxInt) + mem := v_2 + v.reset(OpAMD64LoweredPanicBoundsRC) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCToAux(PanicBoundsC{C: c}) + v.AddArg2(x, mem) + return true + } + // match: (LoweredPanicBoundsRR [kind] (MOVQconst [c]) y mem) + // result: (LoweredPanicBoundsCR [kind] {PanicBoundsC{C:c}} y mem) + for { + kind := auxIntToInt64(v.AuxInt) + if v_0.Op != OpAMD64MOVQconst { + break + } + c := auxIntToInt64(v_0.AuxInt) + y := v_1 + mem := v_2 + v.reset(OpAMD64LoweredPanicBoundsCR) + v.AuxInt = int64ToAuxInt(kind) + v.Aux = panicBoundsCToAux(PanicBoundsC{C: c}) + v.AddArg2(y, mem) + return true + } + return false +} func rewriteValueAMD64_OpAMD64MOVBELstore(v *Value) bool { v_2 := v.Args[2] v_1 := v.Args[1] @@ -27767,60 +27854,6 @@ func rewriteValueAMD64_OpOffPtr(v *Value) bool { return true } } -func rewriteValueAMD64_OpPanicBounds(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (PanicBounds [kind] x y mem) - // cond: boundsABI(kind) == 0 - // result: (LoweredPanicBoundsA [kind] x y mem) - for { - kind := auxIntToInt64(v.AuxInt) - x := v_0 - y := v_1 - mem := v_2 - if !(boundsABI(kind) == 0) { - break - } - v.reset(OpAMD64LoweredPanicBoundsA) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg3(x, y, mem) - return true - } - // match: (PanicBounds [kind] x y mem) - // cond: boundsABI(kind) == 1 - // result: (LoweredPanicBoundsB [kind] x y mem) - for { - kind := auxIntToInt64(v.AuxInt) - x := v_0 - y := v_1 - mem := v_2 - if !(boundsABI(kind) == 1) { - break - } - v.reset(OpAMD64LoweredPanicBoundsB) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg3(x, y, mem) - return true - } - // match: (PanicBounds [kind] x y mem) - // cond: boundsABI(kind) == 2 - // result: (LoweredPanicBoundsC [kind] x y mem) - for { - kind := auxIntToInt64(v.AuxInt) - x := v_0 - y := v_1 - mem := v_2 - if !(boundsABI(kind) == 2) { - break - } - v.reset(OpAMD64LoweredPanicBoundsC) - v.AuxInt = int64ToAuxInt(kind) - v.AddArg3(x, y, mem) - return true - } - return false -} func rewriteValueAMD64_OpPopCount16(v *Value) bool { v_0 := v.Args[0] b := v.Block diff --git a/src/cmd/compile/internal/x86/ssa.go b/src/cmd/compile/internal/x86/ssa.go index 347c5cb560..3347b38b28 100644 --- a/src/cmd/compile/internal/x86/ssa.go +++ b/src/cmd/compile/internal/x86/ssa.go @@ -16,6 +16,7 @@ import ( "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/x86" + "internal/abi" ) // ssaMarkMoves marks any MOVXconst ops that need to avoid clobbering flags. @@ -740,19 +741,165 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { // AuxInt encodes how many buffer entries we need. p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1] - case ssa.Op386LoweredPanicBoundsA, ssa.Op386LoweredPanicBoundsB, ssa.Op386LoweredPanicBoundsC: - p := s.Prog(obj.ACALL) - p.To.Type = obj.TYPE_MEM - p.To.Name = obj.NAME_EXTERN - p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt] - s.UseArgs(8) // space used in callee args area by assembly stubs + case ssa.Op386LoweredPanicBoundsRR, ssa.Op386LoweredPanicBoundsRC, ssa.Op386LoweredPanicBoundsCR, ssa.Op386LoweredPanicBoundsCC, + ssa.Op386LoweredPanicExtendRR, ssa.Op386LoweredPanicExtendRC: + // Compute the constant we put in the PCData entry for this call. + code, signed := ssa.BoundsKind(v.AuxInt).Code() + xIsReg := false + yIsReg := false + xVal := 0 + yVal := 0 + extend := false + switch v.Op { + case ssa.Op386LoweredPanicBoundsRR: + xIsReg = true + xVal = int(v.Args[0].Reg() - x86.REG_AX) + yIsReg = true + yVal = int(v.Args[1].Reg() - x86.REG_AX) + case ssa.Op386LoweredPanicExtendRR: + extend = true + xIsReg = true + hi := int(v.Args[0].Reg() - x86.REG_AX) + lo := int(v.Args[1].Reg() - x86.REG_AX) + xVal = hi<<2 + lo // encode 2 register numbers + yIsReg = true + yVal = int(v.Args[2].Reg() - x86.REG_AX) + case ssa.Op386LoweredPanicBoundsRC: + xIsReg = true + xVal = int(v.Args[0].Reg() - x86.REG_AX) + c := v.Aux.(ssa.PanicBoundsC).C + if c >= 0 && c <= abi.BoundsMaxConst { + yVal = int(c) + } else { + // Move constant to a register + yIsReg = true + if yVal == xVal { + yVal = 1 + } + p := s.Prog(x86.AMOVL) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c + p.To.Type = obj.TYPE_REG + p.To.Reg = x86.REG_AX + int16(yVal) + } + case ssa.Op386LoweredPanicExtendRC: + extend = true + xIsReg = true + hi := int(v.Args[0].Reg() - x86.REG_AX) + lo := int(v.Args[1].Reg() - x86.REG_AX) + xVal = hi<<2 + lo // encode 2 register numbers + c := v.Aux.(ssa.PanicBoundsC).C + if c >= 0 && c <= abi.BoundsMaxConst { + yVal = int(c) + } else { + // Move constant to a register + for yVal == hi || yVal == lo { + yVal++ + } + p := s.Prog(x86.AMOVL) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c + p.To.Type = obj.TYPE_REG + p.To.Reg = x86.REG_AX + int16(yVal) + } + case ssa.Op386LoweredPanicBoundsCR: + yIsReg = true + yVal := int(v.Args[0].Reg() - x86.REG_AX) + c := v.Aux.(ssa.PanicBoundsC).C + if c >= 0 && c <= abi.BoundsMaxConst { + xVal = int(c) + } else if signed && int64(int32(c)) == c || !signed && int64(uint32(c)) == c { + // Move constant to a register + xIsReg = true + if xVal == yVal { + xVal = 1 + } + p := s.Prog(x86.AMOVL) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c + p.To.Type = obj.TYPE_REG + p.To.Reg = x86.REG_AX + int16(xVal) + } else { + // Move constant to two registers + extend = true + xIsReg = true + hi := 0 + lo := 1 + if hi == yVal { + hi = 2 + } + if lo == yVal { + lo = 2 + } + xVal = hi<<2 + lo + p := s.Prog(x86.AMOVL) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c >> 32 + p.To.Type = obj.TYPE_REG + p.To.Reg = x86.REG_AX + int16(hi) + p = s.Prog(x86.AMOVL) + p.From.Type = obj.TYPE_CONST + p.From.Offset = int64(int32(c)) + p.To.Type = obj.TYPE_REG + p.To.Reg = x86.REG_AX + int16(lo) + } + case ssa.Op386LoweredPanicBoundsCC: + c := v.Aux.(ssa.PanicBoundsCC).Cx + if c >= 0 && c <= abi.BoundsMaxConst { + xVal = int(c) + } else if signed && int64(int32(c)) == c || !signed && int64(uint32(c)) == c { + // Move constant to a register + xIsReg = true + p := s.Prog(x86.AMOVL) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c + p.To.Type = obj.TYPE_REG + p.To.Reg = x86.REG_AX + int16(xVal) + } else { + // Move constant to two registers + extend = true + xIsReg = true + hi := 0 + lo := 1 + xVal = hi<<2 + lo + p := s.Prog(x86.AMOVL) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c >> 32 + p.To.Type = obj.TYPE_REG + p.To.Reg = x86.REG_AX + int16(hi) + p = s.Prog(x86.AMOVL) + p.From.Type = obj.TYPE_CONST + p.From.Offset = int64(int32(c)) + p.To.Type = obj.TYPE_REG + p.To.Reg = x86.REG_AX + int16(lo) + } + c = v.Aux.(ssa.PanicBoundsCC).Cy + if c >= 0 && c <= abi.BoundsMaxConst { + yVal = int(c) + } else { + // Move constant to a register + yIsReg = true + yVal = 2 + p := s.Prog(x86.AMOVL) + p.From.Type = obj.TYPE_CONST + p.From.Offset = c + p.To.Type = obj.TYPE_REG + p.To.Reg = x86.REG_AX + int16(yVal) + } + } + c := abi.BoundsEncode(code, signed, xIsReg, yIsReg, xVal, yVal) - case ssa.Op386LoweredPanicExtendA, ssa.Op386LoweredPanicExtendB, ssa.Op386LoweredPanicExtendC: - p := s.Prog(obj.ACALL) + p := s.Prog(obj.APCDATA) + p.From.SetConst(abi.PCDATA_PanicBounds) + p.To.SetConst(int64(c)) + p = s.Prog(obj.ACALL) p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN - p.To.Sym = ssagen.ExtendCheckFunc[v.AuxInt] - s.UseArgs(12) // space used in callee args area by assembly stubs + if extend { + p.To.Sym = ir.Syms.PanicExtend + } else { + p.To.Sym = ir.Syms.PanicBounds + } case ssa.Op386CALLstatic, ssa.Op386CALLclosure, ssa.Op386CALLinter: s.Call(v) diff --git a/src/cmd/internal/obj/x86/obj6.go b/src/cmd/internal/obj/x86/obj6.go index 53c0918254..1208f3d31e 100644 --- a/src/cmd/internal/obj/x86/obj6.go +++ b/src/cmd/internal/obj/x86/obj6.go @@ -653,6 +653,11 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { case obj.ACALL: // Treat common runtime calls that take no arguments // the same as duffcopy and duffzero. + + // Note that of these functions, panicBounds does + // use some stack, but its stack together with the + // < StackSmall used by this function is still + // less than stackNosplit. See issue 31219. if !isZeroArgRuntimeCall(q.To.Sym) { leaf = false break LeafSearch @@ -969,13 +974,7 @@ func isZeroArgRuntimeCall(s *obj.LSym) bool { return false } switch s.Name { - case "runtime.panicdivide", "runtime.panicwrap", "runtime.panicshift": - return true - } - if strings.HasPrefix(s.Name, "runtime.panicIndex") || strings.HasPrefix(s.Name, "runtime.panicSlice") { - // These functions do take arguments (in registers), - // but use no stack before they do a stack check. We - // should include them. See issue 31219. + case "runtime.panicdivide", "runtime.panicwrap", "runtime.panicshift", "runtime.panicBounds", "runtime.panicExtend": return true } return false diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index 62ab83985f..df32e90fda 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -1509,161 +1509,47 @@ TEXT runtime·gcWriteBarrier8(SB),NOSPLIT,$0 MOVL $32, DI JMP gcWriteBarrier<>(SB) -// Note: these functions use a special calling convention to save generated code space. -// Arguments are passed in registers, but the space for those arguments are allocated -// in the caller's stack frame. These stubs write the args into that stack space and -// then tail call to the corresponding runtime handler. -// The tail call makes these stubs disappear in backtraces. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-8 - MOVL AX, x+0(FP) - MOVL CX, y+4(FP) - JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-8 - MOVL AX, x+0(FP) - MOVL CX, y+4(FP) - JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-8 - MOVL CX, x+0(FP) - MOVL DX, y+4(FP) - JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-8 - MOVL CX, x+0(FP) - MOVL DX, y+4(FP) - JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-8 - MOVL CX, x+0(FP) - MOVL DX, y+4(FP) - JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-8 - MOVL CX, x+0(FP) - MOVL DX, y+4(FP) - JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-8 - MOVL AX, x+0(FP) - MOVL CX, y+4(FP) - JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-8 - MOVL AX, x+0(FP) - MOVL CX, y+4(FP) - JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-8 - MOVL DX, x+0(FP) - MOVL BX, y+4(FP) - JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-8 - MOVL DX, x+0(FP) - MOVL BX, y+4(FP) - JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-8 - MOVL DX, x+0(FP) - MOVL BX, y+4(FP) - JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-8 - MOVL DX, x+0(FP) - MOVL BX, y+4(FP) - JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-8 - MOVL CX, x+0(FP) - MOVL DX, y+4(FP) - JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-8 - MOVL CX, x+0(FP) - MOVL DX, y+4(FP) - JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-8 - MOVL AX, x+0(FP) - MOVL CX, y+4(FP) - JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-8 - MOVL AX, x+0(FP) - MOVL CX, y+4(FP) - JMP runtime·goPanicSlice3CU(SB) -TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-8 - MOVL DX, x+0(FP) - MOVL BX, y+4(FP) - JMP runtime·goPanicSliceConvert(SB) - -// Extended versions for 64-bit indexes. -TEXT runtime·panicExtendIndex(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL AX, lo+4(FP) - MOVL CX, y+8(FP) - JMP runtime·goPanicExtendIndex(SB) -TEXT runtime·panicExtendIndexU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL AX, lo+4(FP) - MOVL CX, y+8(FP) - JMP runtime·goPanicExtendIndexU(SB) -TEXT runtime·panicExtendSliceAlen(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL CX, lo+4(FP) - MOVL DX, y+8(FP) - JMP runtime·goPanicExtendSliceAlen(SB) -TEXT runtime·panicExtendSliceAlenU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL CX, lo+4(FP) - MOVL DX, y+8(FP) - JMP runtime·goPanicExtendSliceAlenU(SB) -TEXT runtime·panicExtendSliceAcap(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL CX, lo+4(FP) - MOVL DX, y+8(FP) - JMP runtime·goPanicExtendSliceAcap(SB) -TEXT runtime·panicExtendSliceAcapU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL CX, lo+4(FP) - MOVL DX, y+8(FP) - JMP runtime·goPanicExtendSliceAcapU(SB) -TEXT runtime·panicExtendSliceB(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL AX, lo+4(FP) - MOVL CX, y+8(FP) - JMP runtime·goPanicExtendSliceB(SB) -TEXT runtime·panicExtendSliceBU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL AX, lo+4(FP) - MOVL CX, y+8(FP) - JMP runtime·goPanicExtendSliceBU(SB) -TEXT runtime·panicExtendSlice3Alen(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL DX, lo+4(FP) - MOVL BX, y+8(FP) - JMP runtime·goPanicExtendSlice3Alen(SB) -TEXT runtime·panicExtendSlice3AlenU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL DX, lo+4(FP) - MOVL BX, y+8(FP) - JMP runtime·goPanicExtendSlice3AlenU(SB) -TEXT runtime·panicExtendSlice3Acap(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL DX, lo+4(FP) - MOVL BX, y+8(FP) - JMP runtime·goPanicExtendSlice3Acap(SB) -TEXT runtime·panicExtendSlice3AcapU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL DX, lo+4(FP) - MOVL BX, y+8(FP) - JMP runtime·goPanicExtendSlice3AcapU(SB) -TEXT runtime·panicExtendSlice3B(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL CX, lo+4(FP) - MOVL DX, y+8(FP) - JMP runtime·goPanicExtendSlice3B(SB) -TEXT runtime·panicExtendSlice3BU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL CX, lo+4(FP) - MOVL DX, y+8(FP) - JMP runtime·goPanicExtendSlice3BU(SB) -TEXT runtime·panicExtendSlice3C(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL AX, lo+4(FP) - MOVL CX, y+8(FP) - JMP runtime·goPanicExtendSlice3C(SB) -TEXT runtime·panicExtendSlice3CU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL AX, lo+4(FP) - MOVL CX, y+8(FP) - JMP runtime·goPanicExtendSlice3CU(SB) +TEXT runtime·panicBounds(SB),NOSPLIT,$40-0 + NO_LOCAL_POINTERS + // Save all int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + MOVL AX, 8(SP) + MOVL CX, 12(SP) + MOVL DX, 16(SP) + MOVL BX, 20(SP) + // skip SP @ 24(SP) + MOVL BP, 28(SP) + MOVL SI, 32(SP) + MOVL DI, 36(SP) + + MOVL SP, AX // hide SP read from vet + MOVL 40(AX), AX // PC immediately after call to panicBounds + MOVL AX, 0(SP) + LEAL 8(SP), AX + MOVL AX, 4(SP) + CALL runtime·panicBounds32(SB) + RET + +TEXT runtime·panicExtend(SB),NOSPLIT,$40-0 + NO_LOCAL_POINTERS + // Save all int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + MOVL AX, 8(SP) + MOVL CX, 12(SP) + MOVL DX, 16(SP) + MOVL BX, 20(SP) + // skip SP @ 24(SP) + MOVL BP, 28(SP) + MOVL SI, 32(SP) + MOVL DI, 36(SP) + + MOVL SP, AX // hide SP read from vet + MOVL 40(AX), AX // PC immediately after call to panicExtend + MOVL AX, 0(SP) + LEAL 8(SP), AX + MOVL AX, 4(SP) + CALL runtime·panicBounds32X(SB) + RET #ifdef GOOS_android // Use the free TLS_SLOT_APP slot #2 on Android Q. diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 8983eeafcb..cf1d49a4ad 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -2024,69 +2024,32 @@ TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16 BYTE $0xcc RET -// Note: these functions use a special calling convention to save generated code space. -// Arguments are passed in registers, but the space for those arguments are allocated -// in the caller's stack frame. These stubs write the args into that stack space and -// then tail call to the corresponding runtime handler. -// The tail call makes these stubs disappear in backtraces. -// Defined as ABIInternal since they do not use the stack-based Go ABI. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 - MOVQ CX, BX - JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 - MOVQ CX, BX - JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 - MOVQ CX, AX - MOVQ DX, BX - JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 - MOVQ CX, AX - MOVQ DX, BX - JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 - MOVQ CX, AX - MOVQ DX, BX - JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 - MOVQ CX, AX - MOVQ DX, BX - JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 - MOVQ CX, BX - JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 - MOVQ CX, BX - JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 - MOVQ DX, AX - JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 - MOVQ DX, AX - JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 - MOVQ DX, AX - JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 - MOVQ DX, AX - JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 - MOVQ CX, AX - MOVQ DX, BX - JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 - MOVQ CX, AX - MOVQ DX, BX - JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 - MOVQ CX, BX - JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 - MOVQ CX, BX - JMP runtime·goPanicSlice3CU(SB) -TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16 - MOVQ DX, AX - JMP runtime·goPanicSliceConvert(SB) +TEXT runtime·panicBounds(SB),NOSPLIT,$144-0 + NO_LOCAL_POINTERS + // Save all 14 int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + MOVQ AX, 16(SP) + MOVQ CX, 24(SP) + MOVQ DX, 32(SP) + MOVQ BX, 40(SP) + // skip SP @ 48(SP) + MOVQ BP, 56(SP) + MOVQ SI, 64(SP) + MOVQ DI, 72(SP) + MOVQ R8, 80(SP) + MOVQ R9, 88(SP) + MOVQ R10, 96(SP) + MOVQ R11, 104(SP) + MOVQ R12, 112(SP) + MOVQ R13, 120(SP) + // skip R14 @ 128(SP) (aka G) + MOVQ R15, 136(SP) + + MOVQ SP, AX // hide SP read from vet + MOVQ 152(AX), AX // PC immediately after call to panicBounds + LEAQ 16(SP), BX + CALL runtime·panicBounds64(SB) + RET #ifdef GOOS_android // Use the free TLS_SLOT_APP slot #2 on Android Q. -- 2.51.0