cmd/gc: contiguous loop layout

Drop expecttaken function in favor of extra argument
to gbranch and bgen. Mark loop condition as likely to
be true, so that loops are generated inline.

The main benefit here is contiguous code when trying
to read the generated assembly. It has only minor effects
on the timing, and they mostly cancel the minor effects
that aligning function entry points had.  One exception:
both changes made Fannkuch faster.

Compared to before CL 6244066 (before aligned functions)
benchmark                 old ns/op    new ns/op    delta
BenchmarkBinaryTree17    4222117400   4201958800   -0.48%
BenchmarkFannkuch11      3462631800   3215908600   -7.13%
BenchmarkGobDecode         20887622     20899164   +0.06%
BenchmarkGobEncode          9548772      9439083   -1.15%
BenchmarkGzip                151687       152060   +0.25%
BenchmarkGunzip                8742         8711   -0.35%
BenchmarkJSONEncode        62730560     62686700   -0.07%
BenchmarkJSONDecode       252569180    252368960   -0.08%
BenchmarkMandelbrot200      5267599      5252531   -0.29%
BenchmarkRevcomp25M       980813500    985248400   +0.45%
BenchmarkTemplate         361259100    357414680   -1.06%

Compared to tip (aligned functions):
benchmark                 old ns/op    new ns/op    delta
BenchmarkBinaryTree17    4140739800   4201958800   +1.48%
BenchmarkFannkuch11      3259914400   3215908600   -1.35%
BenchmarkGobDecode         20620222     20899164   +1.35%
BenchmarkGobEncode          9384886      9439083   +0.58%
BenchmarkGzip                150333       152060   +1.15%
BenchmarkGunzip                8741         8711   -0.34%
BenchmarkJSONEncode        65210990     62686700   -3.87%
BenchmarkJSONDecode       249394860    252368960   +1.19%
BenchmarkMandelbrot200      5273394      5252531   -0.40%
BenchmarkRevcomp25M       996013800    985248400   -1.08%
BenchmarkTemplate         360620840    357414680   -0.89%

R=ken2
CC=golang-dev
https://golang.org/cl/6245069

diff --git a/src/cmd/5g/cgen.c b/src/cmd/5g/cgen.c
index e0e635bd264e55363c2802997c75cc38330c1e6b..a1732d232085f060e1cabd00297bcdc0491996ce 100644 (file)
--- a/src/cmd/5g/cgen.c
+++ b/src/cmd/5g/cgen.c
@@ -191,12 +191,12 @@ cgen(Node *n, Node *res)
        case OGE:
        case OGT:
        case ONOT:
-               p1 = gbranch(AB, T);
+               p1 = gbranch(AB, T, 0);
                p2 = pc;
                gmove(nodbool(1), res);
-               p3 = gbranch(AB, T);
+               p3 = gbranch(AB, T, 0);
                patch(p1, pc);
-               bgen(n, 1, p2);
+               bgen(n, 1, 0, p2);
                gmove(nodbool(0), res);
                patch(p3, pc);
                goto ret;
@@ -311,7 +311,7 @@ cgen(Node *n, Node *res)
                        gmove(&n2, &n3);
                        gcmp(optoas(OCMP, types[tptr]), &n1, &n3);
                        regfree(&n3);
-                       p1 = gbranch(optoas(OEQ, types[tptr]), T);
+                       p1 = gbranch(optoas(OEQ, types[tptr]), T, -1);
 
                        n2 = n1;
                        n2.op = OINDREG;
@@ -353,7 +353,7 @@ cgen(Node *n, Node *res)
                        gmove(&n2, &n3);
                        gcmp(optoas(OCMP, types[tptr]), &n1, &n3);
                        regfree(&n3);
-                       p1 = gbranch(optoas(OEQ, types[tptr]), T);
+                       p1 = gbranch(optoas(OEQ, types[tptr]), T, -1);
 
                        n2 = n1;
                        n2.op = OINDREG;
@@ -506,7 +506,7 @@ cgenindex(Node *n, Node *res)
        regfree(&n2);
        regfree(&n1);
        splitclean();
-       return gbranch(ABNE, T);
+       return gbranch(ABNE, T, -1);
 }
 
 /*
@@ -635,7 +635,7 @@ agen(Node *n, Node *res)
                                        gcmp(optoas(OCMP, types[TUINT32]), &n4, &n5);
                                        regfree(&n4);
                                        regfree(&n5);
-                                       p1 = gbranch(optoas(OGT, types[TUINT32]), T);
+                                       p1 = gbranch(optoas(OGT, types[TUINT32]), T, +1);
                                        ginscall(panicindex, 0);
                                        patch(p1, pc);
                                }
@@ -680,7 +680,7 @@ agen(Node *n, Node *res)
                        }
                        gcmp(optoas(OCMP, types[TUINT32]), &n2, &n4);
                        regfree(&n4);
-                       p1 = gbranch(optoas(OLT, types[TUINT32]), T);
+                       p1 = gbranch(optoas(OLT, types[TUINT32]), T, +1);
                        if(p2)
                                patch(p2, pc);
                        ginscall(panicindex, 0);
@@ -838,7 +838,7 @@ agenr(Node *n, Node *a, Node *res)
 }
 
 void
-gencmp0(Node *n, Type *t, int o, Prog *to)
+gencmp0(Node *n, Type *t, int o, int likely, Prog *to)
 {
        Node n1, n2, n3;
        int a;
@@ -855,7 +855,7 @@ gencmp0(Node *n, Type *t, int o, Prog *to)
        } else
                gins(ATST, &n1, N);
        a = optoas(o, t);
-       patch(gbranch(a, t), to);
+       patch(gbranch(a, t, likely), to);
        regfree(&n1);
 }
 
@@ -864,7 +864,7 @@ gencmp0(Node *n, Type *t, int o, Prog *to)
  *     if(n == true) goto to;
  */
 void
-bgen(Node *n, int true, Prog *to)
+bgen(Node *n, int true, int likely, Prog *to)
 {
        int et, a;
        Node *nl, *nr, *r;
@@ -902,13 +902,13 @@ bgen(Node *n, int true, Prog *to)
                a = ONE;
                if(!true)
                        a = OEQ;
-               gencmp0(n, n->type, a, to);
+               gencmp0(n, n->type, a, likely, to);
                goto ret;
 
        case OLITERAL:
                // need to ask if it is bool?
                if(!true == !n->val.u.bval)
-                       patch(gbranch(AB, T), to);
+                       patch(gbranch(AB, T, 0), to);
                goto ret;
 
        case OANDAND:
@@ -916,12 +916,12 @@ bgen(Node *n, int true, Prog *to)
                        goto caseor;
 
        caseand:
-               p1 = gbranch(AB, T);
-               p2 = gbranch(AB, T);
+               p1 = gbranch(AB, T, 0);
+               p2 = gbranch(AB, T, 0);
                patch(p1, pc);
-               bgen(n->left, !true, p2);
-               bgen(n->right, !true, p2);
-               p1 = gbranch(AB, T);
+               bgen(n->left, !true, -likely, p2);
+               bgen(n->right, !true, -likely, p2);
+               p1 = gbranch(AB, T, 0);
                patch(p1, to);
                patch(p2, pc);
                goto ret;
@@ -931,8 +931,8 @@ bgen(Node *n, int true, Prog *to)
                        goto caseand;
 
        caseor:
-               bgen(n->left, true, to);
-               bgen(n->right, true, to);
+               bgen(n->left, true, likely, to);
+               bgen(n->right, true, likely, to);
                goto ret;
 
        case OEQ:
@@ -954,7 +954,7 @@ bgen(Node *n, int true, Prog *to)
        switch(n->op) {
 
        case ONOT:
-               bgen(nl, !true, to);
+               bgen(nl, !true, likely, to);
                goto ret;
 
        case OEQ:
@@ -967,14 +967,14 @@ bgen(Node *n, int true, Prog *to)
                if(!true) {
                        if(isfloat[nl->type->etype]) {
                                // brcom is not valid on floats when NaN is involved.
-                               p1 = gbranch(AB, T);
-                               p2 = gbranch(AB, T);
+                               p1 = gbranch(AB, T, 0);
+                               p2 = gbranch(AB, T, 0);
                                patch(p1, pc);
                                ll = n->ninit;
                                n->ninit = nil;
-                               bgen(n, 1, p2);
+                               bgen(n, 1, -likely, p2);
                                n->ninit = ll;
-                               patch(gbranch(AB, T), to);
+                               patch(gbranch(AB, T, 0), to);
                                patch(p2, pc);
                                goto ret;
                        }                               
@@ -1002,7 +1002,7 @@ bgen(Node *n, int true, Prog *to)
                        n2 = n1;
                        n2.op = OINDREG;
                        n2.xoffset = Array_array;
-                       gencmp0(&n2, types[tptr], a, to);
+                       gencmp0(&n2, types[tptr], a, likely, to);
                        regfree(&n1);
                        break;
 
@@ -1019,7 +1019,7 @@ bgen(Node *n, int true, Prog *to)
                        nodconst(&tmp, types[tptr], 0);
                        gmove(&tmp, &n3);
                        gcmp(optoas(OCMP, types[tptr]), &n4, &n3);
-                       patch(gbranch(a, types[tptr]), to);
+                       patch(gbranch(a, types[tptr], likely), to);
                        regfree(&n4);
                        regfree(&n3);
                        regfree(&n1);
@@ -1039,7 +1039,7 @@ bgen(Node *n, int true, Prog *to)
                        n2 = n1;
                        n2.op = OINDREG;
                        n2.xoffset = 0;
-                       gencmp0(&n2, types[tptr], a, to);
+                       gencmp0(&n2, types[tptr], a, likely, to);
                        regfree(&n1);
                        break;
 
@@ -1056,7 +1056,7 @@ bgen(Node *n, int true, Prog *to)
                        nodconst(&tmp, types[tptr], 0);
                        gmove(&tmp, &n3);
                        gcmp(optoas(OCMP, types[tptr]), &n4, &n3);
-                       patch(gbranch(a, types[tptr]), to);
+                       patch(gbranch(a, types[tptr], likely), to);
                        regfree(&n1);
                        regfree(&n3);
                        regfree(&n4);
@@ -1065,7 +1065,7 @@ bgen(Node *n, int true, Prog *to)
                }
 
                if(iscomplex[nl->type->etype]) {
-                       complexbool(a, nl, nr, true, to);
+                       complexbool(a, nl, nr, true, likely, to);
                        break;
                }
 
@@ -1080,17 +1080,17 @@ bgen(Node *n, int true, Prog *to)
                                cgen(nr, &n2);
                                nr = &n2;
                        }
-                       cmp64(nl, nr, a, to);
+                       cmp64(nl, nr, a, likely, to);
                        break;
                }
 
                if(nr->op == OLITERAL) {
                        if(isconst(nr, CTINT) &&  mpgetfix(nr->val.u.xval) == 0) {
-                               gencmp0(nl, nl->type, a, to);
+                               gencmp0(nl, nl->type, a, likely, to);
                                break;
                        }
                        if(nr->val.ctype == CTNIL) {
-                               gencmp0(nl, nl->type, a, to);
+                               gencmp0(nl, nl->type, a, likely, to);
                                break;
                        }
                }
@@ -1112,7 +1112,7 @@ bgen(Node *n, int true, Prog *to)
                        cgen(&tmp, &n1);
 
                        gcmp(optoas(OCMP, nr->type), &n1, &n2);
-                       patch(gbranch(a, nr->type), to);
+                       patch(gbranch(a, nr->type, likely), to);
 
                        regfree(&n1);
                        regfree(&n2);
@@ -1133,14 +1133,17 @@ bgen(Node *n, int true, Prog *to)
 
                gcmp(optoas(OCMP, nr->type), &n1, &n2);
                if(isfloat[nl->type->etype]) {
-                       p1 = gbranch(ABVS, nr->type);
-                       patch(gbranch(a, nr->type), to);
-                       if(n->op == ONE)
+                       if(n->op == ONE) {
+                               p1 = gbranch(ABVS, nr->type, likely);
+                               patch(gbranch(a, nr->type, likely), to);
                                patch(p1, to);
-                       else
+                       } else {
+                               p1 = gbranch(ABVS, nr->type, -likely);
+                               patch(gbranch(a, nr->type, likely), to);
                                patch(p1, pc);
+                       }
                } else {
-                       patch(gbranch(a, nr->type), to);
+                       patch(gbranch(a, nr->type, likely), to);
                }
                regfree(&n1);
                regfree(&n2);
@@ -1340,7 +1343,7 @@ sgen(Node *n, Node *res, int64 w)
                p = gins(ACMP, &src, N);
                raddr(&nend, p);
 
-               patch(gbranch(ABNE, T), ploop);
+               patch(gbranch(ABNE, T, 0), ploop);
                regfree(&nend);
        } else {
                while(c-- > 0) {

diff --git a/src/cmd/5g/cgen64.c b/src/cmd/5g/cgen64.c
index 2bd5b0f9035fe316b024c58a4104dd60b3f98e5f..015fcd67bd362945c87078e67c099379f452ac52 100644 (file)
--- a/src/cmd/5g/cgen64.c
+++ b/src/cmd/5g/cgen64.c
@@ -285,7 +285,7 @@ cgen64(Node *n, Node *res)
                        split64(r, &cl, &ch);
                        gmove(&ch, &s);
                        gins(ATST, &s, N);
-                       p6 = gbranch(ABNE, T);
+                       p6 = gbranch(ABNE, T, 0);
                        gmove(&cl, &s);
                        splitclean();
                } else {
@@ -299,7 +299,7 @@ cgen64(Node *n, Node *res)
                p1->scond = C_SCOND_EQ;
                p1 = gins(AMOVW, &bh, &ah);
                p1->scond = C_SCOND_EQ;
-               p2 = gbranch(ABEQ, T);
+               p2 = gbranch(ABEQ, T, 0);
 
                // shift is < 32
                nodconst(&n1, types[TUINT32], 32);
@@ -323,14 +323,14 @@ cgen64(Node *n, Node *res)
                p1->scond = C_SCOND_LO;
 
                //      BLO     end
-               p3 = gbranch(ABLO, T);
+               p3 = gbranch(ABLO, T, 0);
 
                // shift == 32
                p1 = gins(AEOR, &al, &al);
                p1->scond = C_SCOND_EQ;
                p1 = gins(AMOVW, &bl, &ah);
                p1->scond = C_SCOND_EQ;
-               p4 = gbranch(ABEQ, T);
+               p4 = gbranch(ABEQ, T, 0);
 
                // shift is < 64
                nodconst(&n1, types[TUINT32], 64);
@@ -353,7 +353,7 @@ cgen64(Node *n, Node *res)
                p1 = gregshift(AMOVW, &bl, SHIFT_LL, &s, &ah);
                p1->scond = C_SCOND_LO;
 
-               p5 = gbranch(ABLO, T);
+               p5 = gbranch(ABLO, T, 0);
 
                // shift >= 64
                if (p6 != P) patch(p6, pc);
@@ -448,7 +448,7 @@ olsh_break:
                        else
                                p1 = gins(AEOR, &ah, &ah);
                        p1->scond = C_SCOND_NE;
-                       p6 = gbranch(ABNE, T);
+                       p6 = gbranch(ABNE, T, 0);
                        gmove(&cl, &s);
                        splitclean();
                } else {
@@ -462,7 +462,7 @@ olsh_break:
                p1->scond = C_SCOND_EQ;
                p1 = gins(AMOVW, &bh, &ah);
                p1->scond = C_SCOND_EQ;
-               p2 = gbranch(ABEQ, T);
+               p2 = gbranch(ABEQ, T, 0);
 
                // check if shift is < 32
                nodconst(&n1, types[TUINT32], 32);
@@ -491,7 +491,7 @@ olsh_break:
                p1->scond = C_SCOND_LO;
 
                //      BLO     end
-               p3 = gbranch(ABLO, T);
+               p3 = gbranch(ABLO, T, 0);
 
                // shift == 32
                p1 = gins(AMOVW, &bh, &al);
@@ -500,7 +500,7 @@ olsh_break:
                        gshift(AMOVW, &bh, SHIFT_AR, 31, &ah);
                else
                        gins(AEOR, &ah, &ah);
-               p4 = gbranch(ABEQ, T);
+               p4 = gbranch(ABEQ, T, 0);
 
                // check if shift is < 64
                nodconst(&n1, types[TUINT32], 64);
@@ -526,7 +526,7 @@ olsh_break:
                }
 
                //      BLO     end
-               p5 = gbranch(ABLO, T);
+               p5 = gbranch(ABLO, T, 0);
 
                // s >= 64
                if(p6 != P)
@@ -675,7 +675,7 @@ orsh_break:
  * nl is memory; nr is constant or memory.
  */
 void
-cmp64(Node *nl, Node *nr, int op, Prog *to)
+cmp64(Node *nl, Node *nr, int op, int likely, Prog *to)
 {
        Node lo1, hi1, lo2, hi2, r1, r2;
        Prog *br;
@@ -705,14 +705,14 @@ cmp64(Node *nl, Node *nr, int op, Prog *to)
                // cmp lo
                // beq to
                // L:
-               br = gbranch(ABNE, T);
+               br = gbranch(ABNE, T, -likely);
                break;
        case ONE:
                // cmp hi
                // bne to
                // cmp lo
                // bne to
-               patch(gbranch(ABNE, T), to);
+               patch(gbranch(ABNE, T, likely), to);
                break;
        case OGE:
        case OGT:
@@ -722,8 +722,8 @@ cmp64(Node *nl, Node *nr, int op, Prog *to)
                // cmp lo
                // bge to (or bgt to)
                // L:
-               patch(gbranch(optoas(OGT, t), T), to);
-               br = gbranch(optoas(OLT, t), T);
+               patch(gbranch(optoas(OGT, t), T, likely), to);
+               br = gbranch(optoas(OLT, t), T, -likely);
                break;
        case OLE:
        case OLT:
@@ -733,8 +733,8 @@ cmp64(Node *nl, Node *nr, int op, Prog *to)
                // cmp lo
                // ble to (or jlt to)
                // L:
-               patch(gbranch(optoas(OLT, t), T), to);
-               br = gbranch(optoas(OGT, t), T);
+               patch(gbranch(optoas(OLT, t), T, likely), to);
+               br = gbranch(optoas(OGT, t), T, -likely);
                break;
        }
 
@@ -749,7 +749,7 @@ cmp64(Node *nl, Node *nr, int op, Prog *to)
        regfree(&r2);
 
        // jump again
-       patch(gbranch(optoas(op, t), T), to);
+       patch(gbranch(optoas(op, t), T, likely), to);
 
        // point first branch down here if appropriate
        if(br != P)

diff --git a/src/cmd/5g/gg.h b/src/cmd/5g/gg.h
index 4c057caec3a6eefa0cfee0a2fbf7392e8503d257..a9e7ded68351ee27ea604a3957e42d26579cc40d 100644 (file)
--- a/src/cmd/5g/gg.h
+++ b/src/cmd/5g/gg.h
@@ -93,7 +93,6 @@ Prog* cgenindex(Node *, Node *);
 void   igen(Node*, Node*, Node*);
 void agenr(Node *n, Node *a, Node *res);
 vlong  fieldoffset(Type*, Node*);
-void   bgen(Node*, int, Prog*);
 void   sgen(Node*, Node*, int64);
 void   gmove(Node*, Node*);
 Prog*  gins(int, Node*, Node*);
@@ -109,7 +108,7 @@ void        cgen_shift(int, int, Node*, Node*, Node*);
 /*
  * cgen64.c
  */
-void   cmp64(Node*, Node*, int, Prog*);
+void   cmp64(Node*, Node*, int, int, Prog*);
 void   cgen64(Node*, Node*);
 
 /*
@@ -117,7 +116,7 @@ void        cgen64(Node*, Node*);
  */
 void   clearp(Prog*);
 void   proglist(void);
-Prog*  gbranch(int, Type*);
+Prog*  gbranch(int, Type*, int);
 Prog*  prog(int);
 void   gaddoffset(Node*);
 void   gconv(int, int);

diff --git a/src/cmd/5g/ggen.c b/src/cmd/5g/ggen.c
index 5ba4e61662d2796b2357f762160c713d2dde4a1d..091ab77534623ffa61770afcba6980895c2df81d 100644 (file)
--- a/src/cmd/5g/ggen.c
+++ b/src/cmd/5g/ggen.c
@@ -120,7 +120,7 @@ ginscall(Node *f, int proc)
                        nodconst(&con, types[TINT32], 0);
                        p = gins(ACMP, &con, N);
                        p->reg = 0;
-                       patch(gbranch(ABNE, T), retpc);
+                       patch(gbranch(ABNE, T, -1), retpc);
                }
                break;
        }
@@ -564,7 +564,7 @@ cgen_shift(int op, int bounded, Node *nl, Node *nr, Node *res)
 
        // test for shift being 0
        gins(ATST, &n1, N);
-       p3 = gbranch(ABEQ, T);
+       p3 = gbranch(ABEQ, T, -1);
 
        // test and fix up large shifts
        // TODO: if(!bounded), don't emit some of this.
@@ -632,7 +632,7 @@ clearfat(Node *nl)
 
                p = gins(ACMP, &dst, N);
                raddr(&end, p);
-               patch(gbranch(ABNE, T), pl);
+               patch(gbranch(ABNE, T, 0), pl);
 
                regfree(&end);
        } else
@@ -758,13 +758,13 @@ cmpandthrow(Node *nl, Node *nr)
        if(nl == &n2)
                regfree(&n2);
        if(throwpc == nil) {
-               p1 = gbranch(optoas(op, types[TUINT32]), T);
+               p1 = gbranch(optoas(op, types[TUINT32]), T, +1);
                throwpc = pc;
                ginscall(panicslice, 0);
                patch(p1, pc);
        } else {
                op = brcom(op);
-               p1 = gbranch(optoas(op, types[TUINT32]), T);
+               p1 = gbranch(optoas(op, types[TUINT32]), T, -1);
                patch(p1, throwpc);
        }
 }

diff --git a/src/cmd/5g/gsubr.c b/src/cmd/5g/gsubr.c
index d5593062962a4bb05e5ca8fad4ef1242d2130400..69f0cef5009dfddbfc0763e7a13dc338e259c580 100644 (file)
--- a/src/cmd/5g/gsubr.c
+++ b/src/cmd/5g/gsubr.c
@@ -108,13 +108,18 @@ dumpdata(void)
 /*
  * generate a branch.
  * t is ignored.
+ * likely values are for branch prediction:
+ *     -1 unlikely
+ *     0 no opinion
+ *     +1 likely
  */
 Prog*
-gbranch(int as, Type *t)
+gbranch(int as, Type *t, int likely)
 {
        Prog *p;
 
        USED(t);
+       USED(likely);  // TODO: record this for linker
 
        p = prog(as);
        p->to.type = D_BRANCH;
@@ -220,7 +225,7 @@ clearstk(void)
        p3 = p;
        p = gins(ACMP, &dst, N);
        raddr(&end, p);
-       patch(gbranch(ABNE, T), p3);
+       patch(gbranch(ABNE, T, 0), p3);
 
        // continue with original code.
        gins(ANOP, N, N)->link = p2;
@@ -238,7 +243,7 @@ gjmp(Prog *to)
 {
        Prog *p;
 
-       p = gbranch(AB, T);
+       p = gbranch(AB, T, 0);
        if(to != P)
                patch(p, to);
        return p;
@@ -1982,7 +1987,7 @@ oindex:
                cgen(&n2, &n3);
                gcmp(optoas(OCMP, types[TUINT32]), reg1, &n3);
                regfree(&n3);
-               p1 = gbranch(optoas(OLT, types[TUINT32]), T);
+               p1 = gbranch(optoas(OLT, types[TUINT32]), T, +1);
                if(p2)
                        patch(p2, pc);
                ginscall(panicindex, 0);
@@ -2045,7 +2050,7 @@ oindex_const:
                        gcmp(optoas(OCMP, types[TUINT32]), &n4, &n3);
                        regfree(&n4);
                        regfree(&n3);
-                       p1 = gbranch(optoas(OGT, types[TUINT32]), T);
+                       p1 = gbranch(optoas(OGT, types[TUINT32]), T, +1);
                        ginscall(panicindex, 0);
                        patch(p1, pc);
                }

diff --git a/src/cmd/6g/cgen.c b/src/cmd/6g/cgen.c
index f917a13ba5879ef48dde6065a928fda68732f4c0..249fcd71a043909f537c0991b9eb3ca59be3ce21 100644 (file)
--- a/src/cmd/6g/cgen.c
+++ b/src/cmd/6g/cgen.c
@@ -187,12 +187,12 @@ cgen(Node *n, Node *res)
        case OGE:
        case OGT:
        case ONOT:
-               p1 = gbranch(AJMP, T);
+               p1 = gbranch(AJMP, T, 0);
                p2 = pc;
                gmove(nodbool(1), res);
-               p3 = gbranch(AJMP, T);
+               p3 = gbranch(AJMP, T, 0);
                patch(p1, pc);
-               bgen(n, 1, p2);
+               bgen(n, 1, 0, p2);
                gmove(nodbool(0), res);
                patch(p3, pc);
                goto ret;
@@ -299,7 +299,7 @@ cgen(Node *n, Node *res)
 
                        nodconst(&n2, types[tptr], 0);
                        gins(optoas(OCMP, types[tptr]), &n1, &n2);
-                       p1 = gbranch(optoas(OEQ, types[tptr]), T);
+                       p1 = gbranch(optoas(OEQ, types[tptr]), T, 0);
 
                        n2 = n1;
                        n2.op = OINDREG;
@@ -334,7 +334,7 @@ cgen(Node *n, Node *res)
 
                        nodconst(&n2, types[tptr], 0);
                        gins(optoas(OCMP, types[tptr]), &n1, &n2);
-                       p1 = gbranch(optoas(OEQ, types[tptr]), T);
+                       p1 = gbranch(optoas(OEQ, types[tptr]), T, 0);
 
                        n2 = n1;
                        n2.op = OINDREG;
@@ -592,9 +592,8 @@ agen(Node *n, Node *res)
                                        n1.xoffset = Array_nel;
                                        nodconst(&n2, types[TUINT32], v);
                                        gins(optoas(OCMP, types[TUINT32]), &n1, &n2);
-                                       p1 = gbranch(optoas(OGT, types[TUINT32]), T);
-                                       expecttaken(p1, 1);
-                                       ginscall(panicindex, 0);
+                                       p1 = gbranch(optoas(OGT, types[TUINT32]), T, +1);
+                                       ginscall(panicindex, -1);
                                        patch(p1, pc);
                                }
 
@@ -643,11 +642,10 @@ agen(Node *n, Node *res)
                                nodconst(&n1, t, nl->type->bound);
                        }
                        gins(optoas(OCMP, t), &n2, &n1);
-                       p1 = gbranch(optoas(OLT, t), T);
-                       expecttaken(p1, 1);
+                       p1 = gbranch(optoas(OLT, t), T, +1);
                        if(n5.op != OXXX)
                                regfree(&n5);
-                       ginscall(panicindex, 0);
+                       ginscall(panicindex, -1);
                        patch(p1, pc);
                }
 
@@ -803,7 +801,7 @@ igen(Node *n, Node *a, Node *res)
  *     if(n == true) goto to;
  */
 void
-bgen(Node *n, int true, Prog *to)
+bgen(Node *n, int true, int likely, Prog *to)
 {
        int et, a;
        Node *nl, *nr, *l, *r;
@@ -845,14 +843,14 @@ bgen(Node *n, int true, Prog *to)
                a = AJNE;
                if(!true)
                        a = AJEQ;
-               patch(gbranch(a, n->type), to);
+               patch(gbranch(a, n->type, likely), to);
                regfree(&n1);
                goto ret;
 
        case OLITERAL:
                // need to ask if it is bool?
                if(!true == !n->val.u.bval)
-                       patch(gbranch(AJMP, T), to);
+                       patch(gbranch(AJMP, T, likely), to);
                goto ret;
 
        case ONAME:
@@ -863,7 +861,7 @@ bgen(Node *n, int true, Prog *to)
                a = AJNE;
                if(!true)
                        a = AJEQ;
-               patch(gbranch(a, n->type), to);
+               patch(gbranch(a, n->type, likely), to);
                goto ret;
 
        case OANDAND:
@@ -871,12 +869,12 @@ bgen(Node *n, int true, Prog *to)
                        goto caseor;
 
        caseand:
-               p1 = gbranch(AJMP, T);
-               p2 = gbranch(AJMP, T);
+               p1 = gbranch(AJMP, T, 0);
+               p2 = gbranch(AJMP, T, 0);
                patch(p1, pc);
-               bgen(n->left, !true, p2);
-               bgen(n->right, !true, p2);
-               p1 = gbranch(AJMP, T);
+               bgen(n->left, !true, -likely, p2);
+               bgen(n->right, !true, -likely, p2);
+               p1 = gbranch(AJMP, T, 0);
                patch(p1, to);
                patch(p2, pc);
                goto ret;
@@ -886,8 +884,8 @@ bgen(Node *n, int true, Prog *to)
                        goto caseand;
 
        caseor:
-               bgen(n->left, true, to);
-               bgen(n->right, true, to);
+               bgen(n->left, true, likely, to);
+               bgen(n->right, true, likely, to);
                goto ret;
 
        case OEQ:
@@ -910,7 +908,7 @@ bgen(Node *n, int true, Prog *to)
        switch(n->op) {
 
        case ONOT:
-               bgen(nl, !true, to);
+               bgen(nl, !true, likely, to);
                goto ret;
 
        case OEQ:
@@ -923,14 +921,14 @@ bgen(Node *n, int true, Prog *to)
                if(!true) {
                        if(isfloat[nr->type->etype]) {
                                // brcom is not valid on floats when NaN is involved.
-                               p1 = gbranch(AJMP, T);
-                               p2 = gbranch(AJMP, T);
+                               p1 = gbranch(AJMP, T, 0);
+                               p2 = gbranch(AJMP, T, 0);
                                patch(p1, pc);
                                ll = n->ninit;   // avoid re-genning ninit
                                n->ninit = nil;
-                               bgen(n, 1, p2);
+                               bgen(n, 1, -likely, p2);
                                n->ninit = ll;
-                               patch(gbranch(AJMP, T), to);
+                               patch(gbranch(AJMP, T, 0), to);
                                patch(p2, pc);
                                goto ret;
                        }                               
@@ -961,7 +959,7 @@ bgen(Node *n, int true, Prog *to)
                        n2.type = types[tptr];
                        nodconst(&tmp, types[tptr], 0);
                        gins(optoas(OCMP, types[tptr]), &n2, &tmp);
-                       patch(gbranch(a, types[tptr]), to);
+                       patch(gbranch(a, types[tptr], likely), to);
                        regfree(&n1);
                        break;
                }
@@ -980,12 +978,12 @@ bgen(Node *n, int true, Prog *to)
                        n2.xoffset = 0;
                        nodconst(&tmp, types[tptr], 0);
                        gins(optoas(OCMP, types[tptr]), &n2, &tmp);
-                       patch(gbranch(a, types[tptr]), to);
+                       patch(gbranch(a, types[tptr], likely), to);
                        regfree(&n1);
                        break;
                }
                if(iscomplex[nl->type->etype]) {
-                       complexbool(a, nl, nr, true, to);
+                       complexbool(a, nl, nr, true, likely, to);
                        break;
                }
 
@@ -1011,7 +1009,7 @@ bgen(Node *n, int true, Prog *to)
 
                if(smallintconst(nr)) {
                        gins(optoas(OCMP, nr->type), &n1, nr);
-                       patch(gbranch(optoas(a, nr->type), nr->type), to);
+                       patch(gbranch(optoas(a, nr->type), nr->type, likely), to);
                        regfree(&n1);
                        break;
                }
@@ -1033,18 +1031,18 @@ bgen(Node *n, int true, Prog *to)
                if(isfloat[nr->type->etype] && (n->op == OEQ || n->op == ONE)) {
                        if(n->op == OEQ) {
                                // neither NE nor P
-                               p1 = gbranch(AJNE, T);
-                               p2 = gbranch(AJPS, T);
-                               patch(gbranch(AJMP, T), to);
+                               p1 = gbranch(AJNE, T, -likely);
+                               p2 = gbranch(AJPS, T, -likely);
+                               patch(gbranch(AJMP, T, 0), to);
                                patch(p1, pc);
                                patch(p2, pc);
                        } else {
                                // either NE or P
-                               patch(gbranch(AJNE, T), to);
-                               patch(gbranch(AJPS, T), to);
+                               patch(gbranch(AJNE, T, likely), to);
+                               patch(gbranch(AJPS, T, likely), to);
                        }
                } else
-                       patch(gbranch(optoas(a, nr->type), nr->type), to);
+                       patch(gbranch(optoas(a, nr->type), nr->type, likely), to);
                regfree(&n1);
                regfree(&n2);
                break;

diff --git a/src/cmd/6g/gg.h b/src/cmd/6g/gg.h
index dcda601073e03f43b0adfdeea61fa13aa412f9e0..2e2ee498f3a31b5078c8d9b28dad7566dc888412 100644 (file)
--- a/src/cmd/6g/gg.h
+++ b/src/cmd/6g/gg.h
@@ -86,7 +86,6 @@ int   gen_as_init(Node*);
 void   agen(Node*, Node*);
 void   igen(Node*, Node*, Node*);
 vlong  fieldoffset(Type*, Node*);
-void   bgen(Node*, int, Prog*);
 void   sgen(Node*, Node*, int64);
 void   gmove(Node*, Node*);
 Prog*  gins(int, Node*, Node*);
@@ -103,8 +102,7 @@ int componentgen(Node*, Node*);
  */
 void   clearp(Prog*);
 void   proglist(void);
-Prog*  gbranch(int, Type*);
-void   expecttaken(Prog*, int);
+Prog*  gbranch(int, Type*, int);
 Prog*  prog(int);
 void   gaddoffset(Node*);
 void   gconv(int, int);
@@ -137,7 +135,6 @@ void        nodfconst(Node*, Type*, Mpflt*);
 int    complexop(Node*, Node*);
 void   complexmove(Node*, Node*);
 void   complexgen(Node*, Node*);
-void   complexbool(int, Node*, Node*, int, Prog*);
 
 /*
  * gobj.c

diff --git a/src/cmd/6g/ggen.c b/src/cmd/6g/ggen.c
index 00a88285ff9c8696084fc5f3a7df485f8e99aeba..de5e818429ababd0476cd7d7e63caae5ba5a5f0e 100644 (file)
--- a/src/cmd/6g/ggen.c
+++ b/src/cmd/6g/ggen.c
@@ -66,8 +66,11 @@ ginscall(Node *f, int proc)
                break;
 
        case 0: // normal call
+       case -1:        // normal call but no return
                p = gins(ACALL, N, f);
                afunclit(&p->to);
+               if(proc == -1)
+                       gins(AUNDEF, N, N);
                break;
 
        case 1: // call in new proc (go)
@@ -88,7 +91,7 @@ ginscall(Node *f, int proc)
                if(proc == 2) {
                        nodreg(&reg, types[TINT64], D_AX);
                        gins(ATESTQ, &reg, &reg);
-                       patch(gbranch(AJNE, T), retpc);
+                       patch(gbranch(AJNE, T, -1), retpc);
                }
                break;
        }
@@ -507,8 +510,7 @@ dodiv(int op, Node *nl, Node *nr, Node *res)
        if(check) {
                nodconst(&n4, t, -1);
                gins(optoas(OCMP, t), &n3, &n4);
-               p1 = gbranch(optoas(ONE, t), T);
-               expecttaken(p1, 1);
+               p1 = gbranch(optoas(ONE, t), T, +1);
                nodconst(&n4, t, -1LL<<(t->width*8-1));
                if(t->width == 8) {
                        n5 = n4;
@@ -516,8 +518,7 @@ dodiv(int op, Node *nl, Node *nr, Node *res)
                        gins(AMOVQ, &n5, &n4);
                }
                gins(optoas(OCMP, t), &ax, &n4);
-               p2 = gbranch(optoas(ONE, t), T);
-               expecttaken(p2, 1);
+               p2 = gbranch(optoas(ONE, t), T, +1);
                if(op == ODIV)
                        gmove(&n4, res);
                if(t->width == 8)
@@ -526,7 +527,7 @@ dodiv(int op, Node *nl, Node *nr, Node *res)
                        nodconst(&n4, t, 0);
                        gmove(&n4, res);
                }
-               p3 = gbranch(AJMP, T);
+               p3 = gbranch(AJMP, T, 0);
                patch(p1, pc);
                patch(p2, pc);
        }
@@ -944,8 +945,7 @@ cgen_shift(int op, int bounded, Node *nl, Node *nr, Node *res)
        if(!bounded) {
                nodconst(&n3, tcount, nl->type->width*8);
                gins(optoas(OCMP, tcount), &n1, &n3);
-               p1 = gbranch(optoas(OLT, tcount), T);
-               expecttaken(p1, 1);
+               p1 = gbranch(optoas(OLT, tcount), T, +1);
                if(op == ORSH && issigned[nl->type->etype]) {
                        nodconst(&n3, types[TUINT32], nl->type->width*8-1);
                        gins(a, &n3, &n2);
@@ -1160,15 +1160,13 @@ cmpandthrow(Node *nl, Node *nr)
        if(n1.op != OXXX)
                regfree(&n1);
        if(throwpc == nil) {
-               p1 = gbranch(optoas(op, t), T);
-               expecttaken(p1, 1);
+               p1 = gbranch(optoas(op, t), T, +1);
                throwpc = pc;
-               ginscall(panicslice, 0);
+               ginscall(panicslice, -1);
                patch(p1, pc);
        } else {
                op = brcom(op);
-               p1 = gbranch(optoas(op, t), T);
-               expecttaken(p1, 0);
+               p1 = gbranch(optoas(op, t), T, -1);
                patch(p1, throwpc);
        }
 }

diff --git a/src/cmd/6g/gsubr.c b/src/cmd/6g/gsubr.c
index c2937b54984e3b8c5df58edef0af4baca903d37a..91c7f4e3ad1491416f3a5340e778672bc612d12a 100644 (file)
--- a/src/cmd/6g/gsubr.c
+++ b/src/cmd/6g/gsubr.c
@@ -103,9 +103,13 @@ dumpdata(void)
 /*
  * generate a branch.
  * t is ignored.
+ * likely values are for branch prediction:
+ *     -1 unlikely
+ *     0 no opinion
+ *     +1 likely
  */
 Prog*
-gbranch(int as, Type *t)
+gbranch(int as, Type *t, int likely)
 {
        Prog *p;
        
@@ -114,19 +118,13 @@ gbranch(int as, Type *t)
        p = prog(as);
        p->to.type = D_BRANCH;
        p->to.branch = P;
+       if(as != AJMP && likely != 0) {
+               p->from.type = D_CONST;
+               p->from.offset = likely > 0;
+       }
        return p;
 }
 
-/*
- * mark branch as expected taken or not.
- */
-void
-expecttaken(Prog *p, int taken)
-{
-       p->from.type = D_CONST;
-       p->from.offset = taken;
-}
-
 /*
  * patch previous branch to jump to to.
  */
@@ -223,7 +221,7 @@ gjmp(Prog *to)
 {
        Prog *p;
 
-       p = gbranch(AJMP, T);
+       p = gbranch(AJMP, T, 0);
        if(to != P)
                patch(p, to);
        return p;
@@ -845,9 +843,9 @@ gmove(Node *f, Node *t)
                regalloc(&r4, types[tt], N);
                gins(optoas(OAS, f->type), f, &r1);
                gins(optoas(OCMP, f->type), &bigf, &r1);
-               p1 = gbranch(optoas(OLE, f->type), T);
+               p1 = gbranch(optoas(OLE, f->type), T, +1);
                gins(a, &r1, &r2);
-               p2 = gbranch(AJMP, T);
+               p2 = gbranch(AJMP, T, 0);
                patch(p1, pc);
                gins(optoas(OAS, f->type), &bigf, &r3);
                gins(optoas(OSUB, f->type), &r3, &r1);
@@ -916,9 +914,9 @@ gmove(Node *f, Node *t)
                regalloc(&r4, f->type, N);
                gmove(f, &r1);
                gins(ACMPQ, &r1, &zero);
-               p1 = gbranch(AJLT, T);
+               p1 = gbranch(AJLT, T, +1);
                gins(a, &r1, &r2);
-               p2 = gbranch(AJMP, T);
+               p2 = gbranch(AJMP, T, 0);
                patch(p1, pc);
                gmove(&r1, &r3);
                gins(ASHRQ, &one, &r3);
@@ -2129,11 +2127,10 @@ oindex:
                        nodconst(&n2, types[TUINT64], l->type->bound);
                }
                gins(optoas(OCMP, t), reg1, &n2);
-               p1 = gbranch(optoas(OLT, t), T);
-               expecttaken(p1, 1);
+               p1 = gbranch(optoas(OLT, t), T, +1);
                if(n4.op != OXXX)
                        regfree(&n4);
-               ginscall(panicindex, 0);
+               ginscall(panicindex, -1);
                patch(p1, pc);
        }
 
@@ -2195,8 +2192,8 @@ oindex_const:
                        n1.xoffset = Array_nel;
                        nodconst(&n2, types[TUINT64], v);
                        gins(optoas(OCMP, types[TUINT32]), &n1, &n2);
-                       p1 = gbranch(optoas(OGT, types[TUINT32]), T);
-                       ginscall(panicindex, 0);
+                       p1 = gbranch(optoas(OGT, types[TUINT32]), T, +1);
+                       ginscall(panicindex, -1);
                        patch(p1, pc);
                }
 
@@ -2239,9 +2236,8 @@ oindex_const_sudo:
                nodconst(&n2, types[TUINT64], v);
                p1 = gins(optoas(OCMP, types[TUINT32]), N, &n2);
                p1->from = *a;
-               p1 = gbranch(optoas(OGT, types[TUINT32]), T);
-               expecttaken(p1, 1);
-               ginscall(panicindex, 0);
+               p1 = gbranch(optoas(OGT, types[TUINT32]), T, +1);
+               ginscall(panicindex, -1);
                patch(p1, pc);
                a->offset -= Array_nel;
        }

diff --git a/src/cmd/6g/reg.c b/src/cmd/6g/reg.c
index d68625bfffa51a53260f58eecbd9137b0c50070d..847d45410d4264fae7bb184f7e0e1da1b3279bb5 100644 (file)
--- a/src/cmd/6g/reg.c
+++ b/src/cmd/6g/reg.c
@@ -1744,7 +1744,7 @@ mark(Prog *firstp)
                p->reg = alive;
                if(p->as != ACALL && p->to.type == D_BRANCH && p->to.branch)
                        mark(p->to.branch);
-               if(p->as == AJMP || p->as == ARET || (p->as == ACALL && noreturn(p)))
+               if(p->as == AJMP || p->as == ARET || p->as == AUNDEF)
                        break;
        }
 }

diff --git a/src/cmd/8g/cgen.c b/src/cmd/8g/cgen.c
index 784fa1de4ecf854cd94ca10468e388f5ae78473f..8f6bfc24d09476d9531f4d2cca68605a3ac362ad 100644 (file)
--- a/src/cmd/8g/cgen.c
+++ b/src/cmd/8g/cgen.c
@@ -196,12 +196,12 @@ cgen(Node *n, Node *res)
        case OGE:
        case OGT:
        case ONOT:
-               p1 = gbranch(AJMP, T);
+               p1 = gbranch(AJMP, T, 0);
                p2 = pc;
                gmove(nodbool(1), res);
-               p3 = gbranch(AJMP, T);
+               p3 = gbranch(AJMP, T, 0);
                patch(p1, pc);
-               bgen(n, 1, p2);
+               bgen(n, 1, 0, p2);
                gmove(nodbool(0), res);
                patch(p3, pc);
                return;
@@ -275,7 +275,7 @@ cgen(Node *n, Node *res)
 
                        nodconst(&n2, types[tptr], 0);
                        gins(optoas(OCMP, types[tptr]), &n1, &n2);
-                       p1 = gbranch(optoas(OEQ, types[tptr]), T);
+                       p1 = gbranch(optoas(OEQ, types[tptr]), T, -1);
 
                        n2 = n1;
                        n2.op = OINDREG;
@@ -309,7 +309,7 @@ cgen(Node *n, Node *res)
 
                        nodconst(&n2, types[tptr], 0);
                        gins(optoas(OCMP, types[tptr]), &n1, &n2);
-                       p1 = gbranch(optoas(OEQ, types[tptr]), T);
+                       p1 = gbranch(optoas(OEQ, types[tptr]), T, -1);
 
                        n2 = n1;
                        n2.op = OINDREG;
@@ -471,7 +471,7 @@ cgenindex(Node *n, Node *res)
        nodconst(&zero, types[TINT32], 0);
        gins(ACMPL, &hi, &zero);
        splitclean();
-       return gbranch(AJNE, T);
+       return gbranch(AJNE, T, +1);
 }
                
 /*
@@ -595,9 +595,8 @@ agen(Node *n, Node *res)
                                        n1.xoffset = Array_nel;
                                        nodconst(&n2, types[TUINT32], v);
                                        gins(optoas(OCMP, types[TUINT32]), &n1, &n2);
-                                       p1 = gbranch(optoas(OGT, types[TUINT32]), T);
-                                       expecttaken(p1, 1);
-                                       ginscall(panicindex, 0);
+                                       p1 = gbranch(optoas(OGT, types[TUINT32]), T, +1);
+                                       ginscall(panicindex, -1);
                                        patch(p1, pc);
                                }
 
@@ -633,11 +632,10 @@ agen(Node *n, Node *res)
                        } else
                                nodconst(&n1, types[TUINT32], nl->type->bound);
                        gins(optoas(OCMP, types[TUINT32]), &n2, &n1);
-                       p1 = gbranch(optoas(OLT, types[TUINT32]), T);
-                       expecttaken(p1, 1);
+                       p1 = gbranch(optoas(OLT, types[TUINT32]), T, +1);
                        if(p2)
                                patch(p2, pc);
-                       ginscall(panicindex, 0);
+                       ginscall(panicindex, -1);
                        patch(p1, pc);
                }
                
@@ -802,7 +800,7 @@ agenr(Node *n, Node *a, Node *res)
  *     if(n == true) goto to;
  */
 void
-bgen(Node *n, int true, Prog *to)
+bgen(Node *n, int true, int likely, Prog *to)
 {
        int et, a;
        Node *nl, *nr, *r;
@@ -844,14 +842,14 @@ bgen(Node *n, int true, Prog *to)
                a = AJNE;
                if(!true)
                        a = AJEQ;
-               patch(gbranch(a, n->type), to);
+               patch(gbranch(a, n->type, likely), to);
                regfree(&n1);
                return;
 
        case OLITERAL:
                // need to ask if it is bool?
                if(!true == !n->val.u.bval)
-                       patch(gbranch(AJMP, T), to);
+                       patch(gbranch(AJMP, T, 0), to);
                return;
 
        case ONAME:
@@ -862,7 +860,7 @@ bgen(Node *n, int true, Prog *to)
                a = AJNE;
                if(!true)
                        a = AJEQ;
-               patch(gbranch(a, n->type), to);
+               patch(gbranch(a, n->type, likely), to);
                return;
 
        case OANDAND:
@@ -870,12 +868,12 @@ bgen(Node *n, int true, Prog *to)
                        goto caseor;
 
        caseand:
-               p1 = gbranch(AJMP, T);
-               p2 = gbranch(AJMP, T);
+               p1 = gbranch(AJMP, T, 0);
+               p2 = gbranch(AJMP, T, 0);
                patch(p1, pc);
-               bgen(n->left, !true, p2);
-               bgen(n->right, !true, p2);
-               p1 = gbranch(AJMP, T);
+               bgen(n->left, !true, -likely, p2);
+               bgen(n->right, !true, -likely, p2);
+               p1 = gbranch(AJMP, T, 0);
                patch(p1, to);
                patch(p2, pc);
                return;
@@ -885,8 +883,8 @@ bgen(Node *n, int true, Prog *to)
                        goto caseand;
 
        caseor:
-               bgen(n->left, true, to);
-               bgen(n->right, true, to);
+               bgen(n->left, true, likely, to);
+               bgen(n->right, true, likely, to);
                return;
 
        case OEQ:
@@ -907,7 +905,7 @@ bgen(Node *n, int true, Prog *to)
 
        switch(n->op) {
        case ONOT:
-               bgen(nl, !true, to);
+               bgen(nl, !true, likely, to);
                break;
 
        case OEQ:
@@ -920,14 +918,14 @@ bgen(Node *n, int true, Prog *to)
                if(!true) {
                        if(isfloat[nl->type->etype]) {
                                // brcom is not valid on floats when NaN is involved.
-                               p1 = gbranch(AJMP, T);
-                               p2 = gbranch(AJMP, T);
+                               p1 = gbranch(AJMP, T, 0);
+                               p2 = gbranch(AJMP, T, 0);
                                patch(p1, pc);
                                ll = n->ninit;  // avoid re-genning ninit
                                n->ninit = nil;
-                               bgen(n, 1, p2);
+                               bgen(n, 1, -likely, p2);
                                n->ninit = ll;
-                               patch(gbranch(AJMP, T), to);
+                               patch(gbranch(AJMP, T, 0), to);
                                patch(p2, pc);
                                break;
                        }                               
@@ -958,7 +956,7 @@ bgen(Node *n, int true, Prog *to)
                        n2.type = types[tptr];
                        nodconst(&tmp, types[tptr], 0);
                        gins(optoas(OCMP, types[tptr]), &n2, &tmp);
-                       patch(gbranch(a, types[tptr]), to);
+                       patch(gbranch(a, types[tptr], likely), to);
                        regfree(&n1);
                        break;
                }
@@ -977,7 +975,7 @@ bgen(Node *n, int true, Prog *to)
                        n2.xoffset = 0;
                        nodconst(&tmp, types[tptr], 0);
                        gins(optoas(OCMP, types[tptr]), &n2, &tmp);
-                       patch(gbranch(a, types[tptr]), to);
+                       patch(gbranch(a, types[tptr], likely), to);
                        regfree(&n1);
                        break;
                }
@@ -1024,21 +1022,21 @@ bgen(Node *n, int true, Prog *to)
                        }
                        if(a == OEQ) {
                                // neither NE nor P
-                               p1 = gbranch(AJNE, T);
-                               p2 = gbranch(AJPS, T);
-                               patch(gbranch(AJMP, T), to);
+                               p1 = gbranch(AJNE, T, -likely);
+                               p2 = gbranch(AJPS, T, -likely);
+                               patch(gbranch(AJMP, T, 0), to);
                                patch(p1, pc);
                                patch(p2, pc);
                        } else if(a == ONE) {
                                // either NE or P
-                               patch(gbranch(AJNE, T), to);
-                               patch(gbranch(AJPS, T), to);
+                               patch(gbranch(AJNE, T, likely), to);
+                               patch(gbranch(AJPS, T, likely), to);
                        } else
-                               patch(gbranch(optoas(a, nr->type), T), to);
+                               patch(gbranch(optoas(a, nr->type), T, likely), to);
                        break;
                }
                if(iscomplex[nl->type->etype]) {
-                       complexbool(a, nl, nr, true, to);
+                       complexbool(a, nl, nr, true, likely, to);
                        break;
                }
 
@@ -1053,7 +1051,7 @@ bgen(Node *n, int true, Prog *to)
                                cgen(nr, &n2);
                                nr = &n2;
                        }
-                       cmp64(nl, nr, a, to);
+                       cmp64(nl, nr, a, likely, to);
                        break;
                }
 
@@ -1074,7 +1072,7 @@ bgen(Node *n, int true, Prog *to)
 
                if(smallintconst(nr)) {
                        gins(optoas(OCMP, nr->type), &n1, nr);
-                       patch(gbranch(a, nr->type), to);
+                       patch(gbranch(a, nr->type, likely), to);
                        break;
                }
 
@@ -1085,7 +1083,7 @@ bgen(Node *n, int true, Prog *to)
 
 cmp:
                gins(optoas(OCMP, nr->type), &n1, &n2);
-               patch(gbranch(a, nr->type), to);
+               patch(gbranch(a, nr->type, likely), to);
                regfree(&n2);
                break;
        }

diff --git a/src/cmd/8g/cgen64.c b/src/cmd/8g/cgen64.c
index 3a7de8ab6ffb67a717e5fcf5e9d16c1979777e94..dc50a409b35be44d969091a362ab7721f9433233 100644 (file)
--- a/src/cmd/8g/cgen64.c
+++ b/src/cmd/8g/cgen64.c
@@ -114,9 +114,9 @@ cgen64(Node *n, Node *res)
                // if DX and EX are zero, use 32 x 32 -> 64 unsigned multiply.
                gins(AMOVL, &dx, &fx);
                gins(AORL, &ex, &fx);
-               p1 = gbranch(AJNE, T);
+               p1 = gbranch(AJNE, T, 0);
                gins(AMULL, &cx, N);    // implicit &ax
-               p2 = gbranch(AJMP, T);
+               p2 = gbranch(AJMP, T, 0);
                patch(p1, pc);
 
                // full 64x64 -> 64, from 32x32 -> 64.
@@ -213,7 +213,7 @@ cgen64(Node *n, Node *res)
                p1 = P;
                if(is64(r->type)) {
                        gins(ACMPL, &hi2, ncon(0));
-                       p1 = gbranch(AJNE, T);
+                       p1 = gbranch(AJNE, T, +1);
                        gins(AMOVL, &lo2, &cx);
                } else {
                        cx.type = types[TUINT32];
@@ -222,7 +222,7 @@ cgen64(Node *n, Node *res)
 
                // if shift count is >=64, zero value
                gins(ACMPL, &cx, ncon(64));
-               p2 = gbranch(optoas(OLT, types[TUINT32]), T);
+               p2 = gbranch(optoas(OLT, types[TUINT32]), T, +1);
                if(p1 != P)
                        patch(p1, pc);
                gins(AXORL, &dx, &dx);
@@ -231,11 +231,11 @@ cgen64(Node *n, Node *res)
 
                // if shift count is >= 32, zero low.
                gins(ACMPL, &cx, ncon(32));
-               p1 = gbranch(optoas(OLT, types[TUINT32]), T);
+               p1 = gbranch(optoas(OLT, types[TUINT32]), T, +1);
                gins(AMOVL, &ax, &dx);
                gins(ASHLL, &cx, &dx);  // SHLL only uses bottom 5 bits of count
                gins(AXORL, &ax, &ax);
-               p2 = gbranch(AJMP, T);
+               p2 = gbranch(AJMP, T, 0);
                patch(p1, pc);
 
                // general shift
@@ -302,7 +302,7 @@ cgen64(Node *n, Node *res)
                p1 = P;
                if(is64(r->type)) {
                        gins(ACMPL, &hi2, ncon(0));
-                       p1 = gbranch(AJNE, T);
+                       p1 = gbranch(AJNE, T, +1);
                        gins(AMOVL, &lo2, &cx);
                } else {
                        cx.type = types[TUINT32];
@@ -311,7 +311,7 @@ cgen64(Node *n, Node *res)
 
                // if shift count is >=64, zero or sign-extend value
                gins(ACMPL, &cx, ncon(64));
-               p2 = gbranch(optoas(OLT, types[TUINT32]), T);
+               p2 = gbranch(optoas(OLT, types[TUINT32]), T, +1);
                if(p1 != P)
                        patch(p1, pc);
                if(hi1.type->etype == TINT32) {
@@ -325,7 +325,7 @@ cgen64(Node *n, Node *res)
 
                // if shift count is >= 32, sign-extend hi.
                gins(ACMPL, &cx, ncon(32));
-               p1 = gbranch(optoas(OLT, types[TUINT32]), T);
+               p1 = gbranch(optoas(OLT, types[TUINT32]), T, +1);
                gins(AMOVL, &dx, &ax);
                if(hi1.type->etype == TINT32) {
                        gins(ASARL, &cx, &ax);  // SARL only uses bottom 5 bits of count
@@ -334,7 +334,7 @@ cgen64(Node *n, Node *res)
                        gins(ASHRL, &cx, &ax);
                        gins(AXORL, &dx, &dx);
                }
-               p2 = gbranch(AJMP, T);
+               p2 = gbranch(AJMP, T, 0);
                patch(p1, pc);
 
                // general shift
@@ -462,7 +462,7 @@ out:;
  * nl is memory; nr is constant or memory.
  */
 void
-cmp64(Node *nl, Node *nr, int op, Prog *to)
+cmp64(Node *nl, Node *nr, int op, int likely, Prog *to)
 {
        Node lo1, hi1, lo2, hi2, rr;
        Prog *br;
@@ -492,14 +492,14 @@ cmp64(Node *nl, Node *nr, int op, Prog *to)
                // cmp lo
                // jeq to
                // L:
-               br = gbranch(AJNE, T);
+               br = gbranch(AJNE, T, -likely);
                break;
        case ONE:
                // cmp hi
                // jne to
                // cmp lo
                // jne to
-               patch(gbranch(AJNE, T), to);
+               patch(gbranch(AJNE, T, likely), to);
                break;
        case OGE:
        case OGT:
@@ -509,8 +509,8 @@ cmp64(Node *nl, Node *nr, int op, Prog *to)
                // cmp lo
                // jge to (or jgt to)
                // L:
-               patch(gbranch(optoas(OGT, t), T), to);
-               br = gbranch(optoas(OLT, t), T);
+               patch(gbranch(optoas(OGT, t), T, likely), to);
+               br = gbranch(optoas(OLT, t), T, -likely);
                break;
        case OLE:
        case OLT:
@@ -520,8 +520,8 @@ cmp64(Node *nl, Node *nr, int op, Prog *to)
                // cmp lo
                // jle to (or jlt to)
                // L:
-               patch(gbranch(optoas(OLT, t), T), to);
-               br = gbranch(optoas(OGT, t), T);
+               patch(gbranch(optoas(OLT, t), T, likely), to);
+               br = gbranch(optoas(OGT, t), T, -likely);
                break;
        }
 
@@ -537,7 +537,7 @@ cmp64(Node *nl, Node *nr, int op, Prog *to)
        }
 
        // jump again
-       patch(gbranch(optoas(op, t), T), to);
+       patch(gbranch(optoas(op, t), T, likely), to);
 
        // point first branch down here if appropriate
        if(br != P)

diff --git a/src/cmd/8g/gg.h b/src/cmd/8g/gg.h
index c8a4cdebb5359b70ffe34841c88a5796687694e4..0edb54c178840c30c9b9c0038f0ca26b9e383535 100644 (file)
--- a/src/cmd/8g/gg.h
+++ b/src/cmd/8g/gg.h
@@ -98,7 +98,6 @@ void  agen(Node*, Node*);
 void   agenr(Node *n, Node *a, Node *res);
 void   igen(Node*, Node*, Node*);
 vlong  fieldoffset(Type*, Node*);
-void   bgen(Node*, int, Prog*);
 void   sgen(Node*, Node*, int64);
 void   gmove(Node*, Node*);
 Prog*  gins(int, Node*, Node*);
@@ -113,7 +112,7 @@ void        mfree(Node*);
 /*
  * cgen64.c
  */
-void   cmp64(Node*, Node*, int, Prog*);
+void   cmp64(Node*, Node*, int, int, Prog*);
 void   cgen64(Node*, Node*);
 
 /*
@@ -121,8 +120,7 @@ void        cgen64(Node*, Node*);
  */
 void   clearp(Prog*);
 void   proglist(void);
-Prog*  gbranch(int, Type*);
-void   expecttaken(Prog*, int);
+Prog*  gbranch(int, Type*, int);
 Prog*  prog(int);
 void   gaddoffset(Node*);
 void   gconv(int, int);
@@ -162,7 +160,6 @@ void        nswap(Node*, Node*);
 int    complexop(Node*, Node*);
 void   complexmove(Node*, Node*);
 void   complexgen(Node*, Node*);
-void   complexbool(int, Node*, Node*, int, Prog*);
 
 /*
  * list.c

diff --git a/src/cmd/8g/ggen.c b/src/cmd/8g/ggen.c
index 5fb66f57f005355eef9291c537e1eee8182d4455..b113788ef771c09cc555b70d585dcbc8b724fbd7 100644 (file)
--- a/src/cmd/8g/ggen.c
+++ b/src/cmd/8g/ggen.c
@@ -106,8 +106,11 @@ ginscall(Node *f, int proc)
                break;
 
        case 0: // normal call
+       case -1:        // normal call but no return
                p = gins(ACALL, N, f);
                afunclit(&p->to);
+               if(proc == -1)
+                       gins(AUNDEF, N, N);
                break;
 
        case 1: // call in new proc (go)
@@ -125,7 +128,7 @@ ginscall(Node *f, int proc)
                if(proc == 2) {
                        nodreg(&reg, types[TINT64], D_AX);
                        gins(ATESTL, &reg, &reg);
-                       patch(gbranch(AJNE, T), retpc);
+                       patch(gbranch(AJNE, T, -1), retpc);
                }
                break;
        }
@@ -539,17 +542,17 @@ dodiv(int op, Node *nl, Node *nr, Node *res, Node *ax, Node *dx)
        if(check) {
                nodconst(&n4, t, -1);
                gins(optoas(OCMP, t), &n1, &n4);
-               p1 = gbranch(optoas(ONE, t), T);
+               p1 = gbranch(optoas(ONE, t), T, +1);
                nodconst(&n4, t, -1LL<<(t->width*8-1));
                gins(optoas(OCMP, t), ax, &n4);
-               p2 = gbranch(optoas(ONE, t), T);
+               p2 = gbranch(optoas(ONE, t), T, +1);
                if(op == ODIV)
                        gmove(&n4, res);
                if(op == OMOD) {
                        nodconst(&n4, t, 0);
                        gmove(&n4, res);
                }
-               p3 = gbranch(AJMP, T);
+               p3 = gbranch(AJMP, T, 0);
                patch(p1, pc);
                patch(p2, pc);
        }
@@ -705,13 +708,13 @@ cgen_shift(int op, int bounded, Node *nl, Node *nr, Node *res)
                        split64(&nt, &lo, &hi);
                        gmove(&lo, &n1);
                        gins(optoas(OCMP, types[TUINT32]), &hi, ncon(0));
-                       p2 = gbranch(optoas(ONE, types[TUINT32]), T);
+                       p2 = gbranch(optoas(ONE, types[TUINT32]), T, +1);
                        gins(optoas(OCMP, types[TUINT32]), &n1, ncon(w));
-                       p1 = gbranch(optoas(OLT, types[TUINT32]), T);
+                       p1 = gbranch(optoas(OLT, types[TUINT32]), T, +1);
                        patch(p2, pc);
                } else {
                        gins(optoas(OCMP, nr->type), &n1, ncon(w));
-                       p1 = gbranch(optoas(OLT, types[TUINT32]), T);
+                       p1 = gbranch(optoas(OLT, types[TUINT32]), T, +1);
                }
                if(op == ORSH && issigned[nl->type->etype]) {
                        gins(a, ncon(w-1), &n2);
@@ -895,15 +898,13 @@ cmpandthrow(Node *nl, Node *nr)
        if(n1.op != OXXX)
                regfree(&n1);
        if(throwpc == nil) {
-               p1 = gbranch(optoas(op, t), T);
-               expecttaken(p1, 1);
+               p1 = gbranch(optoas(op, t), T, +1);
                throwpc = pc;
-               ginscall(panicslice, 0);
+               ginscall(panicslice, -1);
                patch(p1, pc);
        } else {
                op = brcom(op);
-               p1 = gbranch(optoas(op, t), T);
-               expecttaken(p1, 0);
+               p1 = gbranch(optoas(op, t), T, -1);
                patch(p1, throwpc);
        }
 }

diff --git a/src/cmd/8g/gsubr.c b/src/cmd/8g/gsubr.c
index 86ed30522afca4760d18405306da924fb40c75f2..943d36e9b976a031abaa1263d96f583b65a1a0b0 100644 (file)
--- a/src/cmd/8g/gsubr.c
+++ b/src/cmd/8g/gsubr.c
@@ -105,9 +105,13 @@ dumpdata(void)
 /*
  * generate a branch.
  * t is ignored.
+ * likely values are for branch prediction:
+ *     -1 unlikely
+ *     0 no opinion
+ *     +1 likely
  */
 Prog*
-gbranch(int as, Type *t)
+gbranch(int as, Type *t, int likely)
 {
        Prog *p;
 
@@ -115,16 +119,13 @@ gbranch(int as, Type *t)
        p = prog(as);
        p->to.type = D_BRANCH;
        p->to.branch = P;
+       if(likely != 0) {
+               p->from.type = D_CONST;
+               p->from.offset = likely > 0;
+       }
        return p;
 }
 
-void
-expecttaken(Prog *p, int taken)
-{
-       p->from.type = D_CONST;
-       p->from.offset = taken;
-}
-
 /*
  * patch previous branch to jump to to.
  */
@@ -221,7 +222,7 @@ gjmp(Prog *to)
 {
        Prog *p;
 
-       p = gbranch(AJMP, T);
+       p = gbranch(AJMP, T, 0);
        if(to != P)
                patch(p, to);
        return p;
@@ -1452,10 +1453,10 @@ gmove(Node *f, Node *t)
                        fatal("gmove %T", t);
                case TINT8:
                        gins(ACMPL, &t1, ncon(-0x80));
-                       p1 = gbranch(optoas(OLT, types[TINT32]), T);
+                       p1 = gbranch(optoas(OLT, types[TINT32]), T, -1);
                        gins(ACMPL, &t1, ncon(0x7f));
-                       p2 = gbranch(optoas(OGT, types[TINT32]), T);
-                       p3 = gbranch(AJMP, T);
+                       p2 = gbranch(optoas(OGT, types[TINT32]), T, -1);
+                       p3 = gbranch(AJMP, T, 0);
                        patch(p1, pc);
                        patch(p2, pc);
                        gmove(ncon(-0x80), &t1);
@@ -1464,14 +1465,14 @@ gmove(Node *f, Node *t)
                        break;
                case TUINT8:
                        gins(ATESTL, ncon(0xffffff00), &t1);
-                       p1 = gbranch(AJEQ, T);
+                       p1 = gbranch(AJEQ, T, +1);
                        gins(AMOVL, ncon(0), &t1);
                        patch(p1, pc);
                        gmove(&t1, t);
                        break;
                case TUINT16:
                        gins(ATESTL, ncon(0xffff0000), &t1);
-                       p1 = gbranch(AJEQ, T);
+                       p1 = gbranch(AJEQ, T, +1);
                        gins(AMOVL, ncon(0), &t1);
                        patch(p1, pc);
                        gmove(&t1, t);
@@ -1486,7 +1487,7 @@ gmove(Node *f, Node *t)
                gmove(f, &t1);
                split64(&t1, &tlo, &thi);
                gins(ACMPL, &thi, ncon(0));
-               p1 = gbranch(AJEQ, T);
+               p1 = gbranch(AJEQ, T, +1);
                gins(AMOVL, ncon(0), &tlo);
                patch(p1, pc);
                gmove(&tlo, t);
@@ -1505,18 +1506,18 @@ gmove(Node *f, Node *t)
                // if 0 > v { answer = 0 }
                gmove(&zerof, &f0);
                gins(AFUCOMIP, &f0, &f1);
-               p1 = gbranch(optoas(OGT, types[tt]), T);
+               p1 = gbranch(optoas(OGT, types[tt]), T, 0);
                // if 1<<64 <= v { answer = 0 too }
                gmove(&two64f, &f0);
                gins(AFUCOMIP, &f0, &f1);
-               p2 = gbranch(optoas(OGT, types[tt]), T);
+               p2 = gbranch(optoas(OGT, types[tt]), T, 0);
                patch(p1, pc);
                gins(AFMOVVP, &f0, t);  // don't care about t, but will pop the stack
                split64(t, &tlo, &thi);
                gins(AMOVL, ncon(0), &tlo);
                gins(AMOVL, ncon(0), &thi);
                splitclean();
-               p1 = gbranch(AJMP, T);
+               p1 = gbranch(AJMP, T, 0);
                patch(p2, pc);
 
                // in range; algorithm is:
@@ -1533,9 +1534,9 @@ gmove(Node *f, Node *t)
                // actual work
                gmove(&two63f, &f0);
                gins(AFUCOMIP, &f0, &f1);
-               p2 = gbranch(optoas(OLE, types[tt]), T);
+               p2 = gbranch(optoas(OLE, types[tt]), T, 0);
                gins(AFMOVVP, &f0, t);
-               p3 = gbranch(AJMP, T);
+               p3 = gbranch(AJMP, T, 0);
                patch(p2, pc);
                gmove(&two63f, &f0);
                gins(AFSUBDP, &f0, &f1);
@@ -1606,11 +1607,11 @@ gmove(Node *f, Node *t)
                split64(&t1, &tlo, &thi);
                gmove(f, &t1);
                gins(ACMPL, &thi, ncon(0));
-               p1 = gbranch(AJLT, T);
+               p1 = gbranch(AJLT, T, 0);
                // native
                t1.type = types[TINT64];
                gmove(&t1, t);
-               p2 = gbranch(AJMP, T);
+               p2 = gbranch(AJMP, T, 0);
                // simulated
                patch(p1, pc);
                gmove(&tlo, &ax);

diff --git a/src/cmd/8g/reg.c b/src/cmd/8g/reg.c
index 29270c8203a3b956bfda50142153194446806df2..50b15b08c4f0efb4480f1c410e0dda2f4e524930 100644 (file)
--- a/src/cmd/8g/reg.c
+++ b/src/cmd/8g/reg.c
@@ -1605,7 +1605,7 @@ mark(Prog *firstp)
                p->reg = alive;
                if(p->as != ACALL && p->to.type == D_BRANCH && p->to.branch)
                        mark(p->to.branch);
-               if(p->as == AJMP || p->as == ARET || (p->as == ACALL && noreturn(p)))
+               if(p->as == AJMP || p->as == ARET || p->as == AUNDEF)
                        break;
        }
 }

diff --git a/src/cmd/gc/cplx.c b/src/cmd/gc/cplx.c
index dea7bc3bbb59fbc8835f25deb8bd22330f393a1b..e0127fc59ae6499ccb23cf37674c12845c9e4a99 100644 (file)
--- a/src/cmd/gc/cplx.c
+++ b/src/cmd/gc/cplx.c
@@ -271,7 +271,7 @@ complexgen(Node *n, Node *res)
 }
 
 void
-complexbool(int op, Node *nl, Node *nr, int true, Prog *to)
+complexbool(int op, Node *nl, Node *nr, int true, int likely, Prog *to)
 {
        Node tnl, tnr;
        Node n1, n2, n3, n4;
@@ -323,7 +323,7 @@ complexbool(int op, Node *nl, Node *nr, int true, Prog *to)
        if(op == ONE)
                true = !true;
 
-       bgen(&na, true, to);
+       bgen(&na, true, likely, to);
 }
 
 void

diff --git a/src/cmd/gc/gen.c b/src/cmd/gc/gen.c
index 694a10ab5ccdc1b0d479d7722830776b8ebc1b25..5012e4a52dee78cbd205661db0a2bf1bc3573742 100644 (file)
--- a/src/cmd/gc/gen.c
+++ b/src/cmd/gc/gen.c
@@ -394,7 +394,7 @@ gen(Node *n)
                }
                gen(n->nincr);                          // contin:      incr
                patch(p1, pc);                          // test:
-               bgen(n->ntest, 0, breakpc);             //              if(!test) goto break
+               bgen(n->ntest, 0, -1, breakpc);         //              if(!test) goto break
                genlist(n->nbody);                              //              body
                gjmp(continpc);
                patch(breakpc, pc);                     // done:
@@ -410,7 +410,7 @@ gen(Node *n)
                p1 = gjmp(P);                   //              goto test
                p2 = gjmp(P);                   // p2:          goto else
                patch(p1, pc);                          // test:
-               bgen(n->ntest, 0, p2);                  //              if(!test) goto p2
+               bgen(n->ntest, 0, 0, p2);                       //              if(!test) goto p2
                genlist(n->nbody);                              //              then
                p3 = gjmp(P);                   //              goto done
                patch(p2, pc);                          // else:

diff --git a/src/cmd/gc/go.h b/src/cmd/gc/go.h
index 5b7b1eba3c9957ae5872ca4b8db3aa8c178b9e98..71268aa6f7dd0082e39a84089e1ea312f0ec6be0 100644 (file)
--- a/src/cmd/gc/go.h
+++ b/src/cmd/gc/go.h
@@ -909,7 +909,7 @@ Mpflt*      truncfltlit(Mpflt *oldv, Type *t);
  *     cplx.c
  */
 void   complexadd(int op, Node *nl, Node *nr, Node *res);
-void   complexbool(int op, Node *nl, Node *nr, int true, Prog *to);
+void   complexbool(int op, Node *nl, Node *nr, int true, int likely, Prog *to);
 void   complexgen(Node *n, Node *res);
 void   complexminus(Node *nl, Node *res);
 void   complexmove(Node *f, Node *t);
@@ -1304,7 +1304,7 @@ EXTERN    Node*   nodfp;
 
 int    anyregalloc(void);
 void   betypeinit(void);
-void   bgen(Node *n, int true, Prog *to);
+void   bgen(Node *n, int true, int likely, Prog *to);
 void   cgen(Node*, Node*);
 void   cgen_asop(Node *n);
 void   cgen_call(Node *n, int proc);