func zerorange(pp *objw.Progs, p *obj.Prog, off, cnt int64, state *uint32) *obj.Prog {
const (
r13 = 1 << iota // if R13 is already zeroed.
- x15 // if X15 is already zeroed. Note: in new ABI, X15 is always zero.
)
if cnt == 0 {
}
p = pp.Append(p, x86.AMOVQ, obj.TYPE_REG, x86.REG_R13, 0, obj.TYPE_MEM, x86.REG_SP, off)
} else if !isPlan9 && cnt <= int64(8*types.RegSize) {
- if !buildcfg.Experiment.RegabiG && *state&x15 == 0 {
- p = pp.Append(p, x86.AXORPS, obj.TYPE_REG, x86.REG_X15, 0, obj.TYPE_REG, x86.REG_X15, 0)
- *state |= x15
- }
-
for i := int64(0); i < cnt/16; i++ {
p = pp.Append(p, x86.AMOVUPS, obj.TYPE_REG, x86.REG_X15, 0, obj.TYPE_MEM, x86.REG_SP, off+i*16)
}
p = pp.Append(p, x86.AMOVUPS, obj.TYPE_REG, x86.REG_X15, 0, obj.TYPE_MEM, x86.REG_SP, off+cnt-int64(16))
}
} else if !isPlan9 && (cnt <= int64(128*types.RegSize)) {
- if !buildcfg.Experiment.RegabiG && *state&x15 == 0 {
- p = pp.Append(p, x86.AXORPS, obj.TYPE_REG, x86.REG_X15, 0, obj.TYPE_REG, x86.REG_X15, 0)
- *state |= x15
- }
// Save DI to r12. With the amd64 Go register abi, DI can contain
// an incoming parameter, whereas R12 is always scratch.
p = pp.Append(p, x86.AMOVQ, obj.TYPE_REG, x86.REG_DI, 0, obj.TYPE_REG, x86.REG_R12, 0)
p.To.Reg = v.Args[0].Reg()
ssagen.AddAux2(&p.To, v, sc.Off64())
case ssa.OpAMD64MOVOstorezero:
- if !buildcfg.Experiment.RegabiG || s.ABI != obj.ABIInternal {
+ if s.ABI != obj.ABIInternal {
// zero X15 manually
opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
}
p.To.Type = obj.TYPE_REG
p.To.Reg = v.Reg()
case ssa.OpAMD64DUFFZERO:
- if !buildcfg.Experiment.RegabiG || s.ABI != obj.ABIInternal {
+ if s.ABI != obj.ABIInternal {
// zero X15 manually
opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
}
// Closure pointer is DX.
ssagen.CheckLoweredGetClosurePtr(v)
case ssa.OpAMD64LoweredGetG:
- if buildcfg.Experiment.RegabiG && s.ABI == obj.ABIInternal {
+ if s.ABI == obj.ABIInternal {
v.Fatalf("LoweredGetG should not appear in ABIInternal")
}
r := v.Reg()
getgFromTLS(s, r)
case ssa.OpAMD64CALLstatic:
- if buildcfg.Experiment.RegabiG && s.ABI == obj.ABI0 && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABIInternal {
+ if s.ABI == obj.ABI0 && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABIInternal {
// zeroing X15 when entering ABIInternal from ABI0
if buildcfg.GOOS != "plan9" { // do not use SSE on Plan 9
opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
getgFromTLS(s, x86.REG_R14)
}
s.Call(v)
- if buildcfg.Experiment.RegabiG && s.ABI == obj.ABIInternal && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABI0 {
+ if s.ABI == obj.ABIInternal && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABI0 {
// zeroing X15 when entering ABIInternal from ABI0
if buildcfg.GOOS != "plan9" { // do not use SSE on Plan 9
opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
case ssa.BlockRet:
s.Prog(obj.ARET)
case ssa.BlockRetJmp:
- if buildcfg.Experiment.RegabiG && s.ABI == obj.ABI0 && b.Aux.(*obj.LSym).ABI() == obj.ABIInternal {
+ if s.ABI == obj.ABI0 && b.Aux.(*obj.LSym).ABI() == obj.ABIInternal {
// zeroing X15 when entering ABIInternal from ABI0
if buildcfg.GOOS != "plan9" { // do not use SSE on Plan 9
opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
c.floatParamRegs = paramFloatRegAMD64
c.FPReg = framepointerRegAMD64
c.LinkReg = linkRegAMD64
- c.hasGReg = buildcfg.Experiment.RegabiG
+ c.hasGReg = true
case "386":
c.PtrSize = 4
c.RegSize = 4
(IsInBounds idx len) => (SETB (CMPQ idx len))
(IsSliceInBounds idx len) => (SETBE (CMPQ idx len))
(NilCheck ...) => (LoweredNilCheck ...)
-(GetG mem) && !(buildcfg.Experiment.RegabiG && v.Block.Func.OwnAux.Fn.ABI() == obj.ABIInternal) => (LoweredGetG mem) // only lower in old ABI. in new ABI we have a G register.
+(GetG mem) && v.Block.Func.OwnAux.Fn.ABI() != obj.ABIInternal => (LoweredGetG mem) // only lower in old ABI. in new ABI we have a G register.
(GetClosurePtr ...) => (LoweredGetClosurePtr ...)
(GetCallerPC ...) => (LoweredGetCallerPC ...)
(GetCallerSP ...) => (LoweredGetCallerSP ...)
package ssa
-import "internal/buildcfg"
import "math"
import "cmd/internal/obj"
import "cmd/compile/internal/types"
func rewriteValueAMD64_OpGetG(v *Value) bool {
v_0 := v.Args[0]
// match: (GetG mem)
- // cond: !(buildcfg.Experiment.RegabiG && v.Block.Func.OwnAux.Fn.ABI() == obj.ABIInternal)
+ // cond: v.Block.Func.OwnAux.Fn.ABI() != obj.ABIInternal
// result: (LoweredGetG mem)
for {
mem := v_0
- if !(!(buildcfg.Experiment.RegabiG && v.Block.Func.OwnAux.Fn.ABI() == obj.ABIInternal)) {
+ if !(v.Block.Func.OwnAux.Fn.ABI() != obj.ABIInternal) {
break
}
v.reset(OpAMD64LoweredGetG)
"cmd/internal/objabi"
"cmd/internal/src"
"cmd/internal/sys"
- "internal/buildcfg"
"log"
"math"
"path"
var regg int16
if !p.From.Sym.NoSplit() || p.From.Sym.Wrapper() {
- if ctxt.Arch.Family == sys.AMD64 && buildcfg.Experiment.RegabiG && cursym.ABI() == obj.ABIInternal {
+ if ctxt.Arch.Family == sys.AMD64 && cursym.ABI() == obj.ABIInternal {
regg = REGG // use the g register directly in ABIInternal
} else {
p = obj.Appendp(p, newprog)
regg = REG_CX
if ctxt.Arch.Family == sys.AMD64 {
- // Using this register means that stacksplit works w/ //go:registerparams even when !buildcfg.Experiment.RegabiG
regg = REGG // == REG_R14
}
p = load_g(ctxt, p, newprog, regg) // load g into regg
// configuration and any variation from this is an experiment.
var experimentBaseline = goexperiment.Flags{
RegabiWrappers: regabiSupported,
- RegabiG: regabiSupported,
RegabiReflect: regabiSupported,
RegabiArgs: regabiSupported,
}
// do the right thing.
names["regabi"] = func(v bool) {
flags.RegabiWrappers = v
- flags.RegabiG = v
flags.RegabiReflect = v
flags.RegabiArgs = v
}
// regabi is only supported on amd64 and arm64.
if GOARCH != "amd64" && GOARCH != "arm64" {
flags.RegabiWrappers = false
- flags.RegabiG = false
flags.RegabiReflect = false
flags.RegabiArgs = false
}
// Check regabi dependencies.
- if flags.RegabiG && !flags.RegabiWrappers {
- Error = fmt.Errorf("GOEXPERIMENT regabig requires regabiwrappers")
- }
- if flags.RegabiArgs && !(flags.RegabiWrappers && flags.RegabiG && flags.RegabiReflect) {
- Error = fmt.Errorf("GOEXPERIMENT regabiargs requires regabiwrappers,regabig,regabireflect")
+ if flags.RegabiArgs && !(flags.RegabiWrappers && flags.RegabiReflect) {
+ Error = fmt.Errorf("GOEXPERIMENT regabiargs requires regabiwrappers,regabireflect")
}
return flags
}
+++ /dev/null
-// Code generated by mkconsts.go. DO NOT EDIT.
-
-//go:build !goexperiment.regabig
-// +build !goexperiment.regabig
-
-package goexperiment
-
-const RegabiG = false
-const RegabiGInt = 0
+++ /dev/null
-// Code generated by mkconsts.go. DO NOT EDIT.
-
-//go:build goexperiment.regabig
-// +build goexperiment.regabig
-
-package goexperiment
-
-const RegabiG = true
-const RegabiGInt = 1
// ABI0 and ABIInternal functions. Without this, the ABIs are
// assumed to be identical so cross-ABI calls are direct.
RegabiWrappers bool
- // RegabiG enables dedicated G and zero registers in
- // ABIInternal.
- //
- // Requires wrappers because it makes the ABIs incompatible.
- RegabiG bool
// RegabiReflect enables the register-passing paths in
// reflection calls. This is also gated by intArgRegs in
// reflect and runtime (which are disabled by default) so it
// or else unwinding from systemstack_switch is incorrect.
// Smashes R9.
TEXT gosave_systemstack_switch<>(SB),NOSPLIT,$0
-#ifndef GOEXPERIMENT_regabig
- get_tls(R14)
- MOVQ g(R14), R14
-#endif
MOVQ $runtime·systemstack_switch(SB), R9
MOVQ R9, (g_sched+gobuf_pc)(R14)
LEAQ 8(SP), R9
PXOR X10, X8
PXOR X11, X9
PXOR X9, X8
-#ifdef GOEXPERIMENT_regabig
// X15 must be zero on return
PXOR X15, X15
-#endif
#ifdef GOEXPERIMENT_regabiargs
MOVQ X8, AX // return X8
#else
PXOR X10, X8
PXOR X11, X9
PXOR X9, X8
-#ifdef GOEXPERIMENT_regabig
// X15 must be zero on return
PXOR X15, X15
-#endif
#ifdef GOEXPERIMENT_regabiargs
MOVQ X8, AX // return X8
#else
// signals. It is quite painful to set X15 in the signal context,
// so we do it here.
TEXT ·sigpanic0(SB),NOSPLIT,$0-0
-#ifdef GOEXPERIMENT_regabig
get_tls(R14)
MOVQ g(R14), R14
#ifndef GOOS_plan9
XORPS X15, X15
-#endif
#endif
JMP ·sigpanic<ABIInternal>(SB)
MOVQ R13, 104(SP)
// TODO: Consider passing g.m.p in as an argument so they can be shared
// across a sequence of write barriers.
-#ifdef GOEXPERIMENT_regabig
MOVQ g_m(R14), R13
-#else
- get_tls(R13)
- MOVQ g(R13), R13
- MOVQ g_m(R13), R13
-#endif
MOVQ m_p(R13), R13
MOVQ (p_wbBuf+wbBuf_next)(R13), R12
// Increment wbBuf.next position.
JE _8
CMPQ BX, $16
JBE _9through16
-#ifndef GOEXPERIMENT_regabig
- PXOR X15, X15
-#endif
CMPQ BX, $32
JBE _17through32
CMPQ BX, $64
MOVOU X13, -48(DI)(BX*1)
MOVOU X14, -32(DI)(BX*1)
MOVOU X15, -16(DI)(BX*1)
-#ifdef GOEXPERIMENT_regabig
// X15 must be zero on return
PXOR X15, X15
-#endif
RET
move_256through2048:
SUBQ $256, BX
LEAQ 256(SI), SI
LEAQ 256(DI), DI
JGE move_256through2048
-#ifdef GOEXPERIMENT_regabig
// X15 must be zero on return
PXOR X15, X15
-#endif
JMP tail
avxUnaligned:
// If addr (RARG1) is out of range, do nothing.
// Otherwise, setup goroutine context and invoke racecall. Other arguments already set.
TEXT racecalladdr<>(SB), NOSPLIT, $0-0
-#ifndef GOEXPERIMENT_regabig
- get_tls(R12)
- MOVQ g(R12), R14
-#endif
MOVQ g_racectx(R14), RARG0 // goroutine context
// Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend).
CMPQ RARG1, runtime·racearenastart(SB)
// R11 = caller's return address
TEXT racefuncenter<>(SB), NOSPLIT, $0-0
MOVQ DX, BX // save function entry context (for closures)
-#ifndef GOEXPERIMENT_regabig
- get_tls(R12)
- MOVQ g(R12), R14
-#endif
MOVQ g_racectx(R14), RARG0 // goroutine context
MOVQ R11, RARG1
// void __tsan_func_enter(ThreadState *thr, void *pc);
// func runtime·racefuncexit()
// Called from instrumented code.
TEXT runtime·racefuncexit(SB), NOSPLIT, $0-0
-#ifndef GOEXPERIMENT_regabig
- get_tls(R12)
- MOVQ g(R12), R14
-#endif
MOVQ g_racectx(R14), RARG0 // goroutine context
// void __tsan_func_exit(ThreadState *thr);
MOVQ $__tsan_func_exit(SB), AX
JAE racecallatomic_ignore
racecallatomic_ok:
// Addr is within the good range, call the atomic function.
-#ifndef GOEXPERIMENT_regabig
- get_tls(R12)
- MOVQ g(R12), R14
-#endif
MOVQ g_racectx(R14), RARG0 // goroutine context
MOVQ 8(SP), RARG1 // caller pc
MOVQ (SP), RARG2 // pc
// An attempt to synchronize on the address would cause crash.
MOVQ AX, BX // remember the original function
MOVQ $__tsan_go_ignore_sync_begin(SB), AX
-#ifndef GOEXPERIMENT_regabig
- get_tls(R12)
- MOVQ g(R12), R14
-#endif
MOVQ g_racectx(R14), RARG0 // goroutine context
CALL racecall<>(SB)
MOVQ BX, AX // restore the original function
// Switches SP to g0 stack and calls (AX). Arguments already set.
TEXT racecall<>(SB), NOSPLIT, $0-0
-#ifndef GOEXPERIMENT_regabig
- get_tls(R12)
- MOVQ g(R12), R14
-#endif
MOVQ g_m(R14), R13
// Switch to g0 stack.
MOVQ SP, R12 // callee-saved, preserved across the CALL
MOVQ SP, R12 // Save old SP; R12 unchanged by C code.
-#ifdef GOEXPERIMENT_regabig
MOVQ g_m(R14), BX // BX unchanged by C code.
-#else
- get_tls(CX)
- MOVQ g(CX), AX
- MOVQ g_m(AX), BX // BX unchanged by C code.
-#endif
// Set vdsoPC and vdsoSP for SIGPROF traceback.
// Save the old values on stack and restore them on exit,
MOVQ CX, m_vdsoPC(BX)
MOVQ DX, m_vdsoSP(BX)
-#ifdef GOEXPERIMENT_regabig
CMPQ R14, m_curg(BX) // Only switch if on curg.
-#else
- CMPQ AX, m_curg(BX) // Only switch if on curg.
-#endif
JNE noswitch
MOVQ m_g0(BX), DX
TEXT time·now(SB),NOSPLIT,$16-24
MOVQ SP, R12 // Save old SP; R12 unchanged by C code.
-#ifdef GOEXPERIMENT_regabig
MOVQ g_m(R14), BX // BX unchanged by C code.
-#else
- get_tls(CX)
- MOVQ g(CX), AX
- MOVQ g_m(AX), BX // BX unchanged by C code.
-#endif
// Store CLOCK_REALTIME results directly to return space.
LEAQ sec+0(FP), SI
MOVQ CX, m_vdsoPC(BX)
MOVQ SI, m_vdsoSP(BX)
-#ifdef GOEXPERIMENT_regabig
CMPQ R14, m_curg(BX) // Only switch if on curg.
-#else
- CMPQ AX, m_curg(BX) // Only switch if on curg.
-#endif
JNE noswitch
MOVQ m_g0(BX), DX