runtime: use hardware divider to improve performance

The hardware divider is an optional component of ARMv7. This patch
detects whether it is available in runtime and use it or not.

1. The hardware divider is detected at startup and a flag is set/clear
   according to a perticular bit of runtime.hwcap.
2. Each call of runtime.udiv will check this flag and decide if
   use the hardware division instruction.

A rough test shows the performance improves 40-50% for ARMv7. And
the compatibility of ARMv5/v6 is not broken.

fixes #19118

Change-Id: Ic586bc9659ebc169553ca2004d2bdb721df823ac
Reviewed-on: https://go-review.googlesource.com/37496
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>

diff --git a/misc/cgo/testcshared/main0.c b/misc/cgo/testcshared/main0.c
index 1274b8950ebef787ed16f8c7e487ad7a58af4cb5..39ef7e30513f3ab5684eac58c279d67a9d9536c6 100644 (file)
--- a/misc/cgo/testcshared/main0.c
+++ b/misc/cgo/testcshared/main0.c
@@ -12,6 +12,7 @@
 //   int8_t DidInitRun();
 //   int8_t DidMainRun();
 //   int32_t FromPkg();
+//   uint32_t Divu(uint32_t, uint32_t);
 int main(void) {
   int8_t ran_init = DidInitRun();
   if (!ran_init) {
@@ -30,6 +31,11 @@ int main(void) {
     fprintf(stderr, "ERROR: FromPkg=%d, want %d\n", from_pkg, 1024);
     return 1;
   }
+  uint32_t divu = Divu(2264, 31);
+  if (divu != 73) {
+    fprintf(stderr, "ERROR: Divu(2264, 31)=%d, want %d\n", divu, 73);
+    return 1;
+  }
   // test.bash looks for "PASS" to ensure this program has reached the end. 
   printf("PASS\n");
   return 0;

diff --git a/misc/cgo/testcshared/src/p/p.go b/misc/cgo/testcshared/src/p/p.go
index 82b445c12109cb3a01f3ec59a2534d5d97768a3e..fb4b5ca8d1a26179335a4cd50520b79ffd933d63 100644 (file)
--- a/misc/cgo/testcshared/src/p/p.go
+++ b/misc/cgo/testcshared/src/p/p.go
@@ -8,3 +8,5 @@ import "C"
 
 //export FromPkg
 func FromPkg() int32 { return 1024 }
+//export Divu
+func Divu(a, b uint32) uint32 { return a / b }

diff --git a/misc/cgo/testshared/shared_test.go b/misc/cgo/testshared/shared_test.go
index 5017570ba60e36d0117108c064eb8846d2b49f6e..a7cec9b2e8aa001b30abb73578f7c11bc7088ead 100644 (file)
--- a/misc/cgo/testshared/shared_test.go
+++ b/misc/cgo/testshared/shared_test.go
@@ -400,6 +400,12 @@ func TestTrivialExecutablePIE(t *testing.T) {
        AssertHasRPath(t, "./trivial.pie", gorootInstallDir)
 }
 
+// Build a division test program and check it runs.
+func TestDivisionExecutable(t *testing.T) {
+       goCmd(t, "install", "-linkshared", "division")
+       run(t, "division executable", "./bin/division")
+}
+
 // Build an executable that uses cgo linked against the shared runtime and check it
 // runs.
 func TestCgoExecutable(t *testing.T) {

diff --git a/misc/cgo/testshared/src/division/division.go b/misc/cgo/testshared/src/division/division.go
new file mode 100644 (file)
index 0000000..a0b11a5
--- /dev/null
+++ b/misc/cgo/testshared/src/division/division.go
@@ -0,0 +1,17 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+//go:noinline
+func div(x, y uint32) uint32 {
+       return x / y
+}
+
+func main() {
+       a := div(97, 11)
+       if a != 8 {
+               panic("FAIL")
+       }
+}
\ No newline at end of file

diff --git a/src/cmd/asm/internal/asm/testdata/arm.s b/src/cmd/asm/internal/asm/testdata/arm.s
index 47a2283f177608c1b4713b855b52e5cb2c5695e7..0ae031ee815ba3913ee24f8e9a4b4921c2d49b08 100644 (file)
--- a/src/cmd/asm/internal/asm/testdata/arm.s
+++ b/src/cmd/asm/internal/asm/testdata/arm.s
@@ -965,6 +965,13 @@ jmp_label_3:
        REVSH   R1, R2               // b12fffe6
        RBIT    R1, R2               // 312fffe6
 
+// DIVHW R0, R1, R2: R1 / R0 -> R2
+       DIVHW   R0, R1, R2           // 11f012e7
+       DIVUHW  R0, R1, R2           // 11f032e7
+// DIVHW R0, R1: R1 / R0 -> R1
+       DIVHW   R0, R1               // 11f011e7
+       DIVUHW  R0, R1               // 11f031e7
+
 //
 // END
 //

diff --git a/src/cmd/internal/obj/arm/a.out.go b/src/cmd/internal/obj/arm/a.out.go
index 8b43984a7a026577c2d0f385eaf3708478a107b7..35875d0b535ee756a6d6db2677ddecf6c1ce5abf 100644 (file)
--- a/src/cmd/internal/obj/arm/a.out.go
+++ b/src/cmd/internal/obj/arm/a.out.go
@@ -247,6 +247,8 @@ const (
        ADIV
        AMOD
        AMODU
+       ADIVHW
+       ADIVUHW
 
        AMOVB
        AMOVBS

diff --git a/src/cmd/internal/obj/arm/anames.go b/src/cmd/internal/obj/arm/anames.go
index 4ee1835628d3983ae868b8708413071ce63c09b8..63cc5da393a9d356468ce2b0baaedb10e78198cb 100644 (file)
--- a/src/cmd/internal/obj/arm/anames.go
+++ b/src/cmd/internal/obj/arm/anames.go
@@ -71,6 +71,8 @@ var Anames = []string{
        "DIV",
        "MOD",
        "MODU",
+       "DIVHW",
+       "DIVUHW",
        "MOVB",
        "MOVBS",
        "MOVBU",

diff --git a/src/cmd/internal/obj/arm/asm5.go b/src/cmd/internal/obj/arm/asm5.go
index 0636193cc8dae52b1c56b7df33a19bb138e433e7..4b91281346f3c18d1d9e3aebf80f1d2b164de2e4 100644 (file)
--- a/src/cmd/internal/obj/arm/asm5.go
+++ b/src/cmd/internal/obj/arm/asm5.go
@@ -142,6 +142,8 @@ var optab = []Optab{
        {AMUL, C_REG, C_NONE, C_REG, 15, 4, 0, 0, 0},
        {ADIV, C_REG, C_REG, C_REG, 16, 4, 0, 0, 0},
        {ADIV, C_REG, C_NONE, C_REG, 16, 4, 0, 0, 0},
+       {ADIVHW, C_REG, C_REG, C_REG, 105, 4, 0, 0, 0},
+       {ADIVHW, C_REG, C_NONE, C_REG, 105, 4, 0, 0, 0},
        {AMULL, C_REG, C_REG, C_REGREG, 17, 4, 0, 0, 0},
        {AMULA, C_REG, C_REG, C_REGREG2, 17, 4, 0, 0, 0},
        {AMOVW, C_REG, C_NONE, C_SAUTO, 20, 4, REGSP, 0, 0},
@@ -1401,6 +1403,9 @@ func buildop(ctxt *obj.Link) {
                        opset(AMODU, r0)
                        opset(ADIVU, r0)
 
+               case ADIVHW:
+                       opset(ADIVUHW, r0)
+
                case AMOVW,
                        AMOVB,
                        AMOVBS,
@@ -2407,6 +2412,16 @@ func (c *ctxt5) asmout(p *obj.Prog, o *Optab, out []uint32) {
                if p.As == ADATABUNDLE {
                        o1 = 0xe125be70
                }
+
+       case 105: /* divhw r,[r,]r */
+               o1 = c.oprrr(p, p.As, int(p.Scond))
+               rf := int(p.From.Reg)
+               rt := int(p.To.Reg)
+               r := int(p.Reg)
+               if r == 0 {
+                       r = rt
+               }
+               o1 |= (uint32(rf)&15)<<8 | (uint32(r)&15)<<0 | (uint32(rt)&15)<<16
        }
 
        out[0] = o1
@@ -2445,6 +2460,10 @@ func (c *ctxt5) oprrr(p *obj.Prog, a obj.As, sc int) uint32 {
                c.ctxt.Diag(".nil/.W on dp instruction")
        }
        switch a {
+       case ADIVHW:
+               return o | 0x71<<20 | 0xf<<12 | 0x1<<4
+       case ADIVUHW:
+               return o | 0x73<<20 | 0xf<<12 | 0x1<<4
        case AMMUL:
                return o | 0x75<<20 | 0xf<<12 | 0x1<<4
        case AMULS:

diff --git a/src/runtime/os_darwin_arm.go b/src/runtime/os_darwin_arm.go
index ee1bd174f1be6d9c1518e910870b5e16b5775cbd..8eb5655969c71d1fd80d7636ccdd6a36f8cf2dbb 100644 (file)
--- a/src/runtime/os_darwin_arm.go
+++ b/src/runtime/os_darwin_arm.go
@@ -4,6 +4,8 @@
 
 package runtime
 
+var hardDiv bool // TODO: set if a hardware divider is available
+
 func checkgoarm() {
        // TODO(minux): FP checks like in os_linux_arm.go.
 

diff --git a/src/runtime/os_freebsd_arm.go b/src/runtime/os_freebsd_arm.go
index 0399499a4ef01df8db1af192e7911925e90cffc6..6e2bc97470cd71a37c7183e5d9e5bf9d1f393075 100644 (file)
--- a/src/runtime/os_freebsd_arm.go
+++ b/src/runtime/os_freebsd_arm.go
@@ -4,6 +4,8 @@
 
 package runtime
 
+var hardDiv bool // TODO: set if a hardware divider is available
+
 func checkgoarm() {
        // TODO(minux): FP checks like in os_linux_arm.go.
 

diff --git a/src/runtime/os_linux_arm.go b/src/runtime/os_linux_arm.go
index 896ec15e6a83547eeb63848f8fdc68594fc0c1f3..7c925d74b5a2b2eae1caff8385805842ee97aaae 100644 (file)
--- a/src/runtime/os_linux_arm.go
+++ b/src/runtime/os_linux_arm.go
@@ -11,11 +11,13 @@ const (
 
        _HWCAP_VFP   = 1 << 6  // introduced in at least 2.6.11
        _HWCAP_VFPv3 = 1 << 13 // introduced in 2.6.30
+       _HWCAP_IDIVA = 1 << 17
 )
 
 var randomNumber uint32
 var armArch uint8 = 6 // we default to ARMv6
 var hwcap uint32      // set by setup_auxv
+var hardDiv bool      // set if a hardware divider is available
 
 func checkgoarm() {
        // On Android, /proc/self/auxv might be unreadable and hwcap won't
@@ -53,6 +55,7 @@ func archauxv(tag, val uintptr) {
 
        case _AT_HWCAP: // CPU capability bit flags
                hwcap = uint32(val)
+               hardDiv = (hwcap & _HWCAP_IDIVA) != 0
        }
 }
 

diff --git a/src/runtime/os_nacl_arm.go b/src/runtime/os_nacl_arm.go
index 8669ee75b46c9bff76a5b8b9f85596b4b9aee86e..c64ebf31d3562e5766f9eb4e9b8de99483af1941 100644 (file)
--- a/src/runtime/os_nacl_arm.go
+++ b/src/runtime/os_nacl_arm.go
@@ -4,6 +4,8 @@
 
 package runtime
 
+var hardDiv bool // TODO: set if a hardware divider is available
+
 func checkgoarm() {
        // TODO(minux): FP checks like in os_linux_arm.go.
 

diff --git a/src/runtime/os_netbsd_arm.go b/src/runtime/os_netbsd_arm.go
index 95603da64394b02bc3d48c9ce4a88f74f56e50f3..b02e36a73ab461b973a789ee3fbb24c03850a48c 100644 (file)
--- a/src/runtime/os_netbsd_arm.go
+++ b/src/runtime/os_netbsd_arm.go
@@ -6,6 +6,8 @@ package runtime
 
 import "unsafe"
 
+var hardDiv bool // TODO: set if a hardware divider is available
+
 func lwp_mcontext_init(mc *mcontextt, stk unsafe.Pointer, mp *m, gp *g, fn uintptr) {
        // Machine dependent mcontext initialisation for LWP.
        mc.__gregs[_REG_R15] = uint32(funcPC(lwp_tramp))

diff --git a/src/runtime/os_openbsd_arm.go b/src/runtime/os_openbsd_arm.go
index be2e1e9959da60f2074ed78b12c24944f81c9a6b..c318578ab50e242dd783a5d60a468c1d9efb2e95 100644 (file)
--- a/src/runtime/os_openbsd_arm.go
+++ b/src/runtime/os_openbsd_arm.go
@@ -4,6 +4,8 @@
 
 package runtime
 
+var hardDiv bool // TODO: set if a hardware divider is available
+
 func checkgoarm() {
        // TODO(minux): FP checks like in os_linux_arm.go.
 

diff --git a/src/runtime/os_plan9_arm.go b/src/runtime/os_plan9_arm.go
index fdce1e7a352d6122044e3dd0d06ca7c11a44f7e2..1ce0141ce25b8e8cb60c05cb02d05e42b61345c2 100644 (file)
--- a/src/runtime/os_plan9_arm.go
+++ b/src/runtime/os_plan9_arm.go
@@ -4,6 +4,8 @@
 
 package runtime
 
+var hardDiv bool // TODO: set if a hardware divider is available
+
 func checkgoarm() {
        return // TODO(minux)
 }

diff --git a/src/runtime/vlop_arm.s b/src/runtime/vlop_arm.s
index d4c411cda26c81ac21ca5d3aa48614b7201b6d88..6fc325cb9368d1da43ef05a6c019c12c559208ec 100644 (file)
--- a/src/runtime/vlop_arm.s
+++ b/src/runtime/vlop_arm.s
@@ -119,6 +119,10 @@ TEXT runtime·_sfloatpanic(SB),NOSPLIT,$-4
 
 // Be careful: Ra == R11 will be used by the linker for synthesized instructions.
 TEXT udiv(SB),NOSPLIT,$-4
+       MOVBU   runtime·hardDiv(SB), Ra
+       CMP     $0, Ra
+       BNE     udiv_hardware
+
        CLZ     Rq, Rs // find normalizing shift
        MOVW.S  Rq<<Rs, Ra
        MOVW    $fast_udiv_tab<>-64(SB), RM
@@ -154,6 +158,14 @@ TEXT udiv(SB),NOSPLIT,$-4
        ADD.PL  $2, Rq
        RET
 
+// use hardware divider
+udiv_hardware:
+       DIVUHW  Rq, Rr, Rs
+       MUL     Rs, Rq, RM
+       RSB     Rr, RM, Rr
+       MOVW    Rs, Rq
+       RET
+
 udiv_by_large_d:
        // at this point we know d>=2^(31-6)=2^25
        SUB     $4, Ra, Ra