runtime, syscall: use the vdso page on linux x86 for faster syscalls instead of int...

8l: fix handling CALL $(constant) code generated by 8a.
8a,8l: add indirect call instruction: CALL *data(SB).

R=rsc, iant
CC=golang-dev
https://golang.org/cl/4817054

diff --git a/src/cmd/8a/a.y b/src/cmd/8a/a.y
index a8ac773daa594c5cf7d68ac14a66acec276abd49..96976089def83443790e9942845a38527421a4ca 100644 (file)
--- a/src/cmd/8a/a.y
+++ b/src/cmd/8a/a.y
@@ -210,6 +210,13 @@ spec3:     /* JMP/CALL */
                $$.from = nullgen;
                $$.to = $1;
        }
+|      '*' nam
+       {
+               $$.from = nullgen;
+               $$.to = $2;
+               $$.to.index = $2.type;
+               $$.to.type = D_INDIR+D_ADDR;
+       }
 
 spec4: /* NOP */
        nonnon

diff --git a/src/cmd/8l/l.h b/src/cmd/8l/l.h
index 4ee0db9672c9b8fdec909e814f5cff1037e8a6bf..a721f384b84255465175baff220b238f21d4d55b 100644 (file)
--- a/src/cmd/8l/l.h
+++ b/src/cmd/8l/l.h
@@ -209,6 +209,7 @@ enum
        Zbr,
        Zcall,
        Zcallcon,
+       Zcallind,
        Zib_,
        Zib_rp,
        Zibo_m,

diff --git a/src/cmd/8l/optab.c b/src/cmd/8l/optab.c
index 42490b8610c218ac9d54b209841e40c75bd7416a..69602d704a2c7c0738d2d57c2ab51840f57c74c2 100644 (file)
--- a/src/cmd/8l/optab.c
+++ b/src/cmd/8l/optab.c
@@ -260,6 +260,7 @@ uchar       yloop[] =
 uchar  ycall[] =
 {
        Ynone,  Yml,    Zo_m,   2,
+       Ynone,  Ycol,   Zcallind,       2,
        Ynone,  Ybr,    Zcall,  0,
        Ynone,  Yi32,   Zcallcon,       1,
        0
@@ -383,7 +384,7 @@ Optab optab[] =
        { ABTSL,        yml_rl, Pm, 0xab },
        { ABTSW,        yml_rl, Pq, 0xab },
        { ABYTE,        ybyte,  Px, 1 },
-       { ACALL,        ycall,  Px, 0xff,(02),0xe8 },
+       { ACALL,        ycall,  Px, 0xff,(02),0xff,(0x15),0xe8 },
        { ACLC,         ynone,  Px, 0xf8 },
        { ACLD,         ynone,  Px, 0xfc },
        { ACLI,         ynone,  Px, 0xfa },

diff --git a/src/cmd/8l/pass.c b/src/cmd/8l/pass.c
index 2e0990c5a8751e40a9ec0776bd3329b914e91be3..54ea965da2e09fe6ed35264c65d6f1a3b9857a57 100644 (file)
--- a/src/cmd/8l/pass.c
+++ b/src/cmd/8l/pass.c
@@ -307,9 +307,12 @@ patch(void)
                                        p->from.offset = 0;
                                }
                        }
-                       if(p->as == ACALL || (p->as == AJMP && p->to.type != D_BRANCH)) {
+                       if((p->as == ACALL && p->to.type != D_BRANCH) || (p->as == AJMP && p->to.type != D_BRANCH)) {
                                s = p->to.sym;
-                               if(s) {
+                               if(p->to.type == D_INDIR+D_ADDR) {
+                                        /* skip check if this is an indirect call (CALL *symbol(SB)) */
+                                        continue;
+                               } else if(s) {
                                        if(debug['c'])
                                                Bprint(&bso, "%s calls %s\n", TNAME, s->name);
                                        if((s->type&~SSUB) != STEXT) {

diff --git a/src/cmd/8l/span.c b/src/cmd/8l/span.c
index ba193da1c610e10d647c2aaa6ec74c5c2fa5f8a5..81c1d37ebbbd5427bc9c1969362d2f8d49bcb775 100644 (file)
--- a/src/cmd/8l/span.c
+++ b/src/cmd/8l/span.c
@@ -285,6 +285,8 @@ oclass(Adr *a)
                                }
                                return Yxxx;
                        }
+                       //if(a->type == D_INDIR+D_ADDR)
+                       //      print("*Ycol\n");
                        return Ycol;
                }
                return Ym;
@@ -1143,6 +1145,18 @@ found:
                r->add = p->to.offset;
                put4(0);
                break;
+       
+       case Zcallind:
+               *andptr++ = op;
+               *andptr++ = o->op[z+1];
+               r = addrel(cursym);
+               r->off = p->pc + andptr - and;
+               r->type = D_ADDR;
+               r->siz = 4;
+               r->add = p->to.offset;
+               r->sym = p->to.sym;
+               put4(0);
+               break;
 
        case Zbyte:
                v = vaddr(&p->from, &rel);

diff --git a/src/cmd/ld/data.c b/src/cmd/ld/data.c
index e7269169e043956bfa955d7e9d9c0271aa9b6007..a7f61c927bdd3ab1195425d86689dafd23429d3d 100644 (file)
--- a/src/cmd/ld/data.c
+++ b/src/cmd/ld/data.c
@@ -182,7 +182,11 @@ relocsym(Sym *s)
                        o = symaddr(r->sym) + r->add;
                        break;
                case D_PCREL:
-                       o = symaddr(r->sym) + r->add - (s->value + r->off + r->siz);
+                       // r->sym can be null when CALL $(constant) is transformed from absoulte PC to relative PC call.
+                       o = 0;
+                       if(r->sym)
+                               o += symaddr(r->sym);
+                       o += r->add - (s->value + r->off + r->siz);
                        break;
                case D_SIZE:
                        o = r->sym->size + r->add;

diff --git a/src/pkg/runtime/linux/386/rt0.s b/src/pkg/runtime/linux/386/rt0.s
index 223e6d2ea4f9ebf968b6b5733fd82eadc12d0cc3..83149540ec42cb1981a8696ab1e41725b3cef955 100644 (file)
--- a/src/pkg/runtime/linux/386/rt0.s
+++ b/src/pkg/runtime/linux/386/rt0.s
@@ -5,5 +5,13 @@
 // Darwin and Linux use the same linkage to main
 
 TEXT _rt0_386_linux(SB),7,$0
-       JMP     _rt0_386(SB)
+       CALL    runtime·linux_setup_vdso(SB)
+       JMP             _rt0_386(SB)
+
+TEXT _fallback_vdso(SB),7,$0
+       INT     $0x80
+       RET
+
+DATA   runtime·_vdso(SB)/4, $_fallback_vdso(SB)
+GLOBL  runtime·_vdso(SB), $4
 

diff --git a/src/pkg/runtime/linux/386/signal.c b/src/pkg/runtime/linux/386/signal.c
index 8916e10bd1822fb2e18873cf10b93aea654d3afb..4045b2efc3a4bc53e213ae55e913e16afe2cf53f 100644 (file)
--- a/src/pkg/runtime/linux/386/signal.c
+++ b/src/pkg/runtime/linux/386/signal.c
@@ -182,3 +182,27 @@ os·sigpipe(void)
        sigaction(SIGPIPE, SIG_DFL, false);
        runtime·raisesigpipe();
 }
+
+#define AT_NULL                0
+#define AT_SYSINFO     32
+extern uint32 runtime·_vdso;
+
+#pragma textflag 7
+void runtime·linux_setup_vdso(int32 argc, void *argv_list)
+{
+       byte **argv = &argv_list;
+       byte **envp;
+       uint32 *auxv;
+
+       // skip envp to get to ELF auxiliary vector.
+       for(envp = &argv[argc+1]; *envp != nil; envp++)
+               ;
+       envp++;
+       
+       for(auxv=(uint32*)envp; auxv[0] != AT_NULL; auxv += 2) {
+               if(auxv[0] == AT_SYSINFO) {
+                       runtime·_vdso = auxv[1];
+                       break;
+               }
+       }               
+}

diff --git a/src/pkg/runtime/linux/386/sys.s b/src/pkg/runtime/linux/386/sys.s
index 0b4a34986c187f5f1d3dce06346606689c4b7726..f87420f788c6dd2657ae9edd60cbeeb231e17442 100644 (file)
--- a/src/pkg/runtime/linux/386/sys.s
+++ b/src/pkg/runtime/linux/386/sys.s
@@ -11,14 +11,14 @@
 TEXT runtime·exit(SB),7,$0
        MOVL    $252, AX        // syscall number
        MOVL    4(SP), BX
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        INT $3  // not reached
        RET
 
 TEXT runtime·exit1(SB),7,$0
        MOVL    $1, AX  // exit - exit the current os thread
        MOVL    4(SP), BX
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        INT $3  // not reached
        RET
 
@@ -27,13 +27,13 @@ TEXT runtime·open(SB),7,$0
        MOVL    4(SP), BX
        MOVL    8(SP), CX
        MOVL    12(SP), DX
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        RET
 
 TEXT runtime·close(SB),7,$0
        MOVL    $6, AX          // syscall - close
        MOVL    4(SP), BX
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        RET
 
 TEXT runtime·write(SB),7,$0
@@ -41,7 +41,7 @@ TEXT runtime·write(SB),7,$0
        MOVL    4(SP), BX
        MOVL    8(SP), CX
        MOVL    12(SP), DX
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        RET
 
 TEXT runtime·read(SB),7,$0
@@ -49,16 +49,16 @@ TEXT runtime·read(SB),7,$0
        MOVL    4(SP), BX
        MOVL    8(SP), CX
        MOVL    12(SP), DX
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        RET
 
 TEXT runtime·raisesigpipe(SB),7,$12
        MOVL    $224, AX        // syscall - gettid
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        MOVL    AX, 0(SP)       // arg 1 tid
        MOVL    $13, 4(SP)      // arg 2 SIGPIPE
        MOVL    $238, AX        // syscall - tkill
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        RET
 
 TEXT runtime·setitimer(SB),7,$0-24
@@ -66,7 +66,7 @@ TEXT runtime·setitimer(SB),7,$0-24
        MOVL    4(SP), BX
        MOVL    8(SP), CX
        MOVL    12(SP), DX
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        RET
 
 TEXT runtime·mincore(SB),7,$0-24
@@ -74,7 +74,7 @@ TEXT runtime·mincore(SB),7,$0-24
        MOVL    4(SP), BX
        MOVL    8(SP), CX
        MOVL    12(SP), DX
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        RET
 
 TEXT runtime·gettime(SB), 7, $32
@@ -82,7 +82,7 @@ TEXT runtime·gettime(SB), 7, $32
        LEAL    8(SP), BX
        MOVL    $0, CX
        MOVL    $0, DX
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
 
        MOVL    8(SP), BX       // sec
        MOVL    sec+0(FP), DI
@@ -100,7 +100,7 @@ TEXT runtime·rt_sigaction(SB),7,$0
        MOVL    8(SP), CX
        MOVL    12(SP), DX
        MOVL    16(SP), SI
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        RET
 
 TEXT runtime·sigtramp(SB),7,$44
@@ -138,7 +138,9 @@ TEXT runtime·sigignore(SB),7,$0
 
 TEXT runtime·sigreturn(SB),7,$0
        MOVL    $173, AX        // rt_sigreturn
-       INT $0x80
+       // Sigreturn expects same SP as signal handler,
+       // so cannot CALL *runtime._vsdo(SB) here.
+       INT     $0x80
        INT $3  // not reached
        RET
 
@@ -151,7 +153,7 @@ TEXT runtime·mmap(SB),7,$0
        MOVL    20(SP), DI
        MOVL    24(SP), BP
        SHRL    $12, BP
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        CMPL    AX, $0xfffff001
        JLS     3(PC)
        NOTL    AX
@@ -162,7 +164,7 @@ TEXT runtime·munmap(SB),7,$0
        MOVL    $91, AX // munmap
        MOVL    4(SP), BX
        MOVL    8(SP), CX
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        CMPL    AX, $0xfffff001
        JLS     2(PC)
        INT $3
@@ -178,7 +180,7 @@ TEXT runtime·futex(SB),7,$0
        MOVL    16(SP), SI
        MOVL    20(SP), DI
        MOVL    24(SP), BP
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        RET
 
 // int32 clone(int32 flags, void *stack, M *m, G *g, void (*fn)(void));
@@ -199,6 +201,10 @@ TEXT runtime·clone(SB),7,$0
        MOVL    SI, 8(CX)
        MOVL    $1234, 12(CX)
 
+       // cannot use CALL *runtime·_vdso(SB) here, because
+       // the stack changes during the system call (after 
+       // CALL *runtime·_vdso(SB), the child is still using
+       // the parent's stack when executing its RET instruction).
        INT     $0x80
 
        // In parent, return.
@@ -214,7 +220,7 @@ TEXT runtime·clone(SB),7,$0
 
        // Initialize AX to Linux tid
        MOVL    $224, AX
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
 
        // In child on new stack.  Reload registers (paranoia).
        MOVL    0(SP), BX       // m
@@ -262,7 +268,7 @@ TEXT runtime·sigaltstack(SB),7,$-8
        MOVL    $186, AX        // sigaltstack
        MOVL    new+4(SP), BX
        MOVL    old+8(SP), CX
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        CMPL    AX, $0xfffff001
        JLS     2(PC)
        INT     $3
@@ -323,7 +329,7 @@ TEXT runtime·setldt(SB),7,$32
        MOVL    AX, CX  // user_desc
        MOVL    $16, DX // sizeof(user_desc)
        MOVL    $123, AX        // syscall - modify_ldt
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
 
        // breakpoint on error
        CMPL AX, $0xfffff001
@@ -340,5 +346,5 @@ TEXT runtime·setldt(SB),7,$32
 
 TEXT runtime·osyield(SB),7,$0
        MOVL    $158, AX
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        RET

diff --git a/src/pkg/syscall/asm_linux_386.s b/src/pkg/syscall/asm_linux_386.s
index 82f170b5bb79423df746f83ae4d1ece8af94fcf7..22e00e45b0e8e387cb5acab02a3a54bedaa3495f 100644 (file)
--- a/src/pkg/syscall/asm_linux_386.s
+++ b/src/pkg/syscall/asm_linux_386.s
@@ -17,7 +17,7 @@ TEXT  ·Syscall(SB),7,$0
        MOVL    16(SP), DX
        MOVL    $0, SI
        MOVL    $0,  DI
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        CMPL    AX, $0xfffff001
        JLS     ok
        MOVL    $-1, 20(SP)     // r1
@@ -43,7 +43,7 @@ TEXT  ·Syscall6(SB),7,$0
        MOVL    20(SP), SI
        MOVL    24(SP), DI
        MOVL    28(SP), BP
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        CMPL    AX, $0xfffff001
        JLS     ok6
        MOVL    $-1, 32(SP)     // r1
@@ -67,7 +67,7 @@ TEXT ·RawSyscall(SB),7,$0
        MOVL    16(SP), DX
        MOVL    $0, SI
        MOVL    $0,  DI
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        CMPL    AX, $0xfffff001
        JLS     ok1
        MOVL    $-1, 20(SP)     // r1
@@ -90,7 +90,7 @@ TEXT  ·RawSyscall6(SB),7,$0
        MOVL    20(SP), SI
        MOVL    24(SP), DI
        MOVL    28(SP), BP
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        CMPL    AX, $0xfffff001
        JLS     ok2
        MOVL    $-1, 32(SP)     // r1
@@ -116,7 +116,7 @@ TEXT ·socketcall(SB),7,$0
        MOVL    $0, DX
        MOVL    $0, SI
        MOVL    $0,  DI
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        CMPL    AX, $0xfffff001
        JLS     oksock
        MOVL    $-1, 32(SP)     // n
@@ -139,7 +139,7 @@ TEXT ·rawsocketcall(SB),7,$0
        MOVL    $0, DX
        MOVL    $0, SI
        MOVL    $0,  DI
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        CMPL    AX, $0xfffff001
        JLS     oksock1
        MOVL    $-1, 32(SP)     // n
@@ -165,7 +165,7 @@ TEXT ·Seek(SB),7,$0
        MOVL    8(SP), DX       // offset-low
        LEAL    20(SP), SI      // result pointer
        MOVL    16(SP),  DI     // whence
-       INT     $0x80
+       CALL    *runtime·_vdso(SB)
        CMPL    AX, $0xfffff001
        JLS     okseek
        MOVL    $-1, 20(SP)     // newoffset low