crypto/aes: simplify key expansion in ppc64le asm

The ported cryptogam implementation uses a subtle and tricky mechanism
using lxv/vperm/lvsl to load unaligned vectors. This is difficult to
read, and may read and write unrelated bytes if reading from an
unaligned address.

Instead, POWER8 instructions can be used to load from unaligned memory
with much less overhead. Alignment interrupts only occur when reading
or writing cache-inhibited memory, which we assume isn't used in go
today, otherwise alignment penalties are usually marginal.

Instead lxvd2x+xxpermdi and xxpermdi+stxvd2x can be used to emulate
unaligned LE bytewise loads, similar to lxv/stxv on POWER9 in
little-endian mode.

Likewise, a custom permute vector is used to emulate BE bytewise
storage operations, lxvb16x/stxvb16x, on POWER9.

This greatly simplifies the code, and it makes it much easier to store
the keys in reverse (which is exactly how the decrypt keys are expected
to be stored).

Change-Id: I2334337e31a8fdf8d13ba96231142a039f237098
Reviewed-on: https://go-review.googlesource.com/c/go/+/395494
Reviewed-by: Lynn Boger <laboger@linux.vnet.ibm.com>
Trust: Paul Murphy <murp@ibm.com>
Run-TryBot: Paul Murphy <murp@ibm.com>
TryBot-Result: Gopher Robot <gobot@golang.org>

diff --git a/src/crypto/aes/asm_ppc64le.s b/src/crypto/aes/asm_ppc64le.s
index 5eae67532299677075b16fa1902182ebc2bed19b..192a8096cd1bc9b4714c762afc4b4d759bc3156c 100644 (file)
--- a/src/crypto/aes/asm_ppc64le.s
+++ b/src/crypto/aes/asm_ppc64le.s
@@ -43,6 +43,10 @@
 #define OUTHEAD V10
 #define OUTTAIL V11
 
+// For P9 instruction emulation
+#define ESPERM  V21  // Endian swapping permute into BE
+#define TMP2    V22  // Temporary for P8_STXVB16X/P8_STXV
+
 // For {en,de}cryptBlockAsm
 #define BLK_INP    R3
 #define BLK_OUT    R4
@@ -50,15 +54,38 @@
 #define BLK_ROUNDS R6
 #define BLK_IDX    R7
 
-DATA ·rcon+0x00(SB)/8, $0x0100000001000000 // RCON
-DATA ·rcon+0x08(SB)/8, $0x0100000001000000 // RCON
-DATA ·rcon+0x10(SB)/8, $0x1b0000001b000000
-DATA ·rcon+0x18(SB)/8, $0x1b0000001b000000
-DATA ·rcon+0x20(SB)/8, $0x0d0e0f0c0d0e0f0c // MASK
-DATA ·rcon+0x28(SB)/8, $0x0d0e0f0c0d0e0f0c // MASK
-DATA ·rcon+0x30(SB)/8, $0x0000000000000000
-DATA ·rcon+0x38(SB)/8, $0x0000000000000000
-GLOBL ·rcon(SB), RODATA, $64
+DATA ·rcon+0x00(SB)/8, $0x0f0e0d0c0b0a0908 // Permute for vector doubleword endian swap
+DATA ·rcon+0x08(SB)/8, $0x0706050403020100
+DATA ·rcon+0x10(SB)/8, $0x0100000001000000 // RCON
+DATA ·rcon+0x18(SB)/8, $0x0100000001000000 // RCON
+DATA ·rcon+0x20(SB)/8, $0x1b0000001b000000
+DATA ·rcon+0x28(SB)/8, $0x1b0000001b000000
+DATA ·rcon+0x30(SB)/8, $0x0d0e0f0c0d0e0f0c // MASK
+DATA ·rcon+0x38(SB)/8, $0x0d0e0f0c0d0e0f0c // MASK
+DATA ·rcon+0x40(SB)/8, $0x0000000000000000
+DATA ·rcon+0x48(SB)/8, $0x0000000000000000
+GLOBL ·rcon(SB), RODATA, $80
+
+// Emulate unaligned BE vector load/stores on LE targets
+#define P8_LXVB16X(RA,RB,VT) \
+       LXVD2X  (RA+RB), VT \
+       VPERM   VT, VT, ESPERM, VT
+
+#define P8_STXVB16X(VS,RA,RB) \
+       VPERM   VS, VS, ESPERM, TMP2 \
+       STXVD2X TMP2, (RA+RB)
+
+#define P8_STXV(VS,RA,RB) \
+       XXPERMDI        VS, VS, $2, TMP2 \
+       STXVD2X         TMP2, (RA+RB)
+
+#define P8_LXV(RA,RB,VT) \
+       LXVD2X          (RA+RB), VT \
+       XXPERMDI        VT, VT, $2, VT
+
+#define LXSDX_BE(RA,RB,VT) \
+       LXSDX   (RA+RB), VT \
+       VPERM   VT, VT, ESPERM, VT
 
 // func setEncryptKeyAsm(key *byte, keylen int, enc *uint32) int
 TEXT ·setEncryptKeyAsm(SB), NOSPLIT|NOFRAME, $0
@@ -87,45 +114,32 @@ TEXT ·doEncryptKeyAsm(SB), NOSPLIT|NOFRAME, $0
        BC      0x06, 2, enc_key_abort // bne-  .Lenc_key_abort
 
        MOVD    $·rcon(SB), PTR // PTR point to rcon addr
+       LVX     (PTR), ESPERM
+       ADD     $0x10, PTR
 
        // Get key from memory and write aligned into VR
-       NEG     INP, R9            // neg   9,3        R9 is ~INP + 1
-       LVX     (INP)(R0), IN0     // lvx   1,0,3      Load key inside IN0
-       ADD     $15, INP, INP      // addi  3,3,15     Add 15B to INP addr
-       LVSR    (R9)(R0), KEY      // lvsr  3,0,9
+       P8_LXVB16X(INP, R0, IN0)
+       ADD     $0x10, INP, INP
        MOVD    $0x20, R8          // li    8,0x20     R8 = 32
+
        CMPW    BITS, $192         // cmpwi 4,192      Key size == 192?
-       LVX     (INP)(R0), IN1     // lvx   2,0,3
-       VSPLTISB        $0x0f, MASK// vspltisb 5,0x0f  0x0f0f0f0f... mask
        LVX     (PTR)(R0), RCON    // lvx   4,0,6      Load first 16 bytes into RCON
-       VXOR    KEY, MASK, KEY     // vxor  3,3,5      Adjust for byte swap
        LVX     (PTR)(R8), MASK    // lvx   5,8,6
        ADD     $0x10, PTR, PTR    // addi  6,6,0x10   PTR to next 16 bytes of RCON
-       VPERM   IN0, IN1, KEY, IN0 // vperm 1,1,2,3    Align
        MOVD    $8, CNT            // li    7,8        CNT = 8
        VXOR    ZERO, ZERO, ZERO   // vxor  0,0,0      Zero to be zero :)
        MOVD    CNT, CTR           // mtctr 7          Set the counter to 8 (rounds)
 
-       LVSL    (OUT)(R0), OUTPERM              // lvsl  8,0,5
-       VSPLTISB        $-1, OUTMASK                    // vspltisb      9,-1
-       LVX     (OUT)(R0), OUTHEAD              // lvx   10,0,5
-       VPERM   OUTMASK, ZERO, OUTPERM, OUTMASK // vperm 9,9,0,8
-
        BLT     loop128      // blt   .Loop128
-       ADD     $8, INP, INP // addi  3,3,8
        BEQ     l192         // beq   .L192
-       ADD     $8, INP, INP // addi  3,3,8
        JMP     l256         // b     .L256
 
 loop128:
        // Key schedule (Round 1 to 8)
        VPERM   IN0, IN0, MASK, KEY              // vperm 3,1,1,5         Rotate-n-splat
        VSLDOI  $12, ZERO, IN0, TMP              // vsldoi 6,0,1,12
-       VPERM   IN0, IN0, OUTPERM, OUTTAIL       // vperm 11,1,1,8    Rotate
-       VSEL    OUTHEAD, OUTTAIL, OUTMASK, STAGE // vsel 7,10,11,9
-       VOR     OUTTAIL, OUTTAIL, OUTHEAD        // vor 10,11,11
+       P8_STXV(IN0, R0, OUT)
        VCIPHERLAST     KEY, RCON, KEY           // vcipherlast 3,3,4
-       STVX    STAGE, (OUT+R0)                  // stvx 7,0,5        Write to output
        ADD     $16, OUT, OUT                    // addi 5,5,16       Point to the next round
 
        VXOR    IN0, TMP, IN0       // vxor 1,1,6
@@ -142,11 +156,8 @@ loop128:
        // Key schedule (Round 9)
        VPERM   IN0, IN0, MASK, KEY              // vperm 3,1,1,5   Rotate-n-spat
        VSLDOI  $12, ZERO, IN0, TMP              // vsldoi 6,0,1,12
-       VPERM   IN0, IN0, OUTPERM, OUTTAIL       // vperm 11,1,1,8  Rotate
-       VSEL    OUTHEAD, OUTTAIL, OUTMASK, STAGE // vsel 7,10,11,9
-       VOR     OUTTAIL, OUTTAIL, OUTHEAD        // vor 10,11,11
+       P8_STXV(IN0, R0, OUT)
        VCIPHERLAST     KEY, RCON, KEY           // vcipherlast 3,3,4
-       STVX    STAGE, (OUT+R0)                  // stvx 7,0,5   Round 9
        ADD     $16, OUT, OUT                    // addi 5,5,16
 
        // Key schedule (Round 10)
@@ -160,11 +171,8 @@ loop128:
 
        VPERM   IN0, IN0, MASK, KEY              // vperm 3,1,1,5   Rotate-n-splat
        VSLDOI  $12, ZERO, IN0, TMP              // vsldoi 6,0,1,12
-       VPERM   IN0, IN0, OUTPERM, OUTTAIL       // vperm 11,1,1,8  Rotate
-       VSEL    OUTHEAD, OUTTAIL, OUTMASK, STAGE // vsel 7,10,11,9
-       VOR     OUTTAIL, OUTTAIL, OUTHEAD        // vor 10,11,11
+       P8_STXV(IN0, R0, OUT)
        VCIPHERLAST     KEY, RCON, KEY           // vcipherlast 3,3,4
-       STVX    STAGE, (OUT+R0)                  // stvx 7,0,5    Round 10
        ADD     $16, OUT, OUT                    // addi 5,5,16
 
        // Key schedule (Round 11)
@@ -174,26 +182,18 @@ loop128:
        VSLDOI  $12, ZERO, TMP, TMP              // vsldoi 6,0,6,12
        VXOR    IN0, TMP, IN0                    // vxor 1,1,6
        VXOR    IN0, KEY, IN0                    // vxor 1,1,3
-       VPERM   IN0, IN0, OUTPERM, OUTTAIL       // vperm 11,1,1,8
-       VSEL    OUTHEAD, OUTTAIL, OUTMASK, STAGE // vsel 7,10,11,9
-       VOR     OUTTAIL, OUTTAIL, OUTHEAD        // vor 10,11,11
-       STVX    STAGE, (OUT+R0)                  // stvx 7,0,5  Round 11
+       P8_STXV(IN0, R0, OUT)
 
-       ADD     $15, OUT, INP   // addi  3,5,15
        ADD     $0x50, OUT, OUT // addi  5,5,0x50
 
        MOVD    $10, ROUNDS // li    8,10
        JMP     done        // b     .Ldone
 
 l192:
-       LVX     (INP)(R0), TMP                   // lvx 6,0,3
+       LXSDX_BE(INP, R0, IN1)                   // Load next 8 bytes into upper half of VSR in BE order.
        MOVD    $4, CNT                          // li 7,4
-       VPERM   IN0, IN0, OUTPERM, OUTTAIL       // vperm 11,1,1,8
-       VSEL    OUTHEAD, OUTTAIL, OUTMASK, STAGE // vsel 7,10,11,9
-       VOR     OUTTAIL, OUTTAIL, OUTHEAD        // vor 10,11,11
-       STVX    STAGE, (OUT+R0)                  // stvx 7,0,5
+       P8_STXV(IN0, R0, OUT)
        ADD     $16, OUT, OUT                    // addi 5,5,16
-       VPERM   IN1, TMP, KEY, IN1               // vperm 2,2,6,3
        VSPLTISB        $8, KEY                  // vspltisb 3,8
        MOVD    CNT, CTR                         // mtctr 7
        VSUBUBM MASK, KEY, MASK                  // vsububm 5,5,3
@@ -221,23 +221,17 @@ loop192:
 
        VPERM   IN1, IN1, MASK, KEY              // vperm 3,2,2,5
        VSLDOI  $12, ZERO, IN0, TMP              // vsldoi 6,0,1,12
-       VPERM   STAGE, STAGE, OUTPERM, OUTTAIL   // vperm 11,7,7,8
-       VSEL    OUTHEAD, OUTTAIL, OUTMASK, STAGE // vsel 7,10,11,9
-       VOR     OUTTAIL, OUTTAIL, OUTHEAD        // vor 10,11,11
+       P8_STXV(STAGE, R0, OUT)
        VCIPHERLAST     KEY, RCON, KEY           // vcipherlast 3,3,4
-       STVX    STAGE, (OUT+R0)                  // stvx 7,0,5
        ADD     $16, OUT, OUT                    // addi 5,5,16
 
        VSLDOI  $8, IN0, IN1, STAGE              // vsldoi 7,1,2,8
        VXOR    IN0, TMP, IN0                    // vxor 1,1,6
        VSLDOI  $12, ZERO, TMP, TMP              // vsldoi 6,0,6,12
-       VPERM   STAGE, STAGE, OUTPERM, OUTTAIL   // vperm 11,7,7,8
-       VSEL    OUTHEAD, OUTTAIL, OUTMASK, STAGE // vsel 7,10,11,9
-       VOR     OUTTAIL, OUTTAIL, OUTHEAD        // vor 10,11,11
+       P8_STXV(STAGE, R0, OUT)
        VXOR    IN0, TMP, IN0                    // vxor 1,1,6
        VSLDOI  $12, ZERO, TMP, TMP              // vsldoi 6,0,6,12
        VXOR    IN0, TMP, IN0                    // vxor 1,1,6
-       STVX    STAGE, (OUT+R0)                  // stvx 7,0,5
        ADD     $16, OUT, OUT                    // addi 5,5,16
 
        VSPLTW  $3, IN0, TMP                     // vspltw 6,1,3
@@ -247,11 +241,7 @@ loop192:
        VXOR    IN1, TMP, IN1                    // vxor 2,2,6
        VXOR    IN0, KEY, IN0                    // vxor 1,1,3
        VXOR    IN1, KEY, IN1                    // vxor 2,2,3
-       VPERM   IN0, IN0, OUTPERM, OUTTAIL       // vperm 11,1,1,8
-       VSEL    OUTHEAD, OUTTAIL, OUTMASK, STAGE // vsel 7,10,11,9
-       VOR     OUTTAIL, OUTTAIL, OUTHEAD        // vor 10,11,11
-       STVX    STAGE, (OUT+R0)                  // stvx 7,0,5
-       ADD     $15, OUT, INP                    // addi 3,5,15
+       P8_STXV(IN0, R0, OUT)
        ADD     $16, OUT, OUT                    // addi 5,5,16
        BC      0x10, 0, loop192                 // bdnz .Loop192
 
@@ -260,25 +250,18 @@ loop192:
        BR      done            // b .Ldone
 
 l256:
-       LVX     (INP)(R0), TMP                   // lvx 6,0,3
+       P8_LXVB16X(INP, R0, IN1)
        MOVD    $7, CNT                          // li 7,7
        MOVD    $14, ROUNDS                      // li 8,14
-       VPERM   IN0, IN0, OUTPERM, OUTTAIL       // vperm 11,1,1,8
-       VSEL    OUTHEAD, OUTTAIL, OUTMASK, STAGE // vsel 7,10,11,9
-       VOR     OUTTAIL, OUTTAIL, OUTHEAD        // vor 10,11,11
-       STVX    STAGE, (OUT+R0)                  // stvx 7,0,5
+       P8_STXV(IN0, R0, OUT)
        ADD     $16, OUT, OUT                    // addi 5,5,16
-       VPERM   IN1, TMP, KEY, IN1               // vperm 2,2,6,3
        MOVD    CNT, CTR                         // mtctr 7
 
 loop256:
        VPERM   IN1, IN1, MASK, KEY              // vperm 3,2,2,5
        VSLDOI  $12, ZERO, IN0, TMP              // vsldoi 6,0,1,12
-       VPERM   IN1, IN1, OUTPERM, OUTTAIL       // vperm 11,2,2,8
-       VSEL    OUTHEAD, OUTTAIL, OUTMASK, STAGE // vsel 7,10,11,9
-       VOR     OUTTAIL, OUTTAIL, OUTHEAD        // vor 10,11,11
+       P8_STXV(IN1, R0, OUT)
        VCIPHERLAST     KEY, RCON, KEY           // vcipherlast 3,3,4
-       STVX    STAGE, (OUT+R0)                  // stvx 7,0,5
        ADD     $16, OUT, OUT                    // addi 5,5,16
 
        VXOR    IN0, TMP, IN0                    // vxor 1,1,6
@@ -288,11 +271,7 @@ loop256:
        VXOR    IN0, TMP, IN0                    // vxor 1,1,6
        VADDUWM RCON, RCON, RCON                 // vadduwm 4,4,4
        VXOR    IN0, KEY, IN0                    // vxor 1,1,3
-       VPERM   IN0, IN0, OUTPERM, OUTTAIL       // vperm 11,1,1,8
-       VSEL    OUTHEAD, OUTTAIL, OUTMASK, STAGE // vsel 7,10,11,9
-       VOR     OUTTAIL, OUTTAIL, OUTHEAD        // vor 10,11,11
-       STVX    STAGE, (OUT+R0)                  // stvx 7,0,5
-       ADD     $15, OUT, INP                    // addi 3,5,15
+       P8_STXV(IN0, R0, OUT)
        ADD     $16, OUT, OUT                    // addi 5,5,16
        BC      0x12, 0, done                    // bdz .Ldone
 
@@ -310,9 +289,6 @@ loop256:
        JMP     loop256       // b .Loop256
 
 done:
-       LVX     (INP)(R0), IN1             // lvx   2,0,3
-       VSEL    OUTHEAD, IN1, OUTMASK, IN1 // vsel 2,10,2,9
-       STVX    IN1, (INP+R0)              // stvx  2,0,3
        MOVD    $0, PTR                    // li    6,0    set PTR to 0 (exit code 0)
        MOVW    ROUNDS, 0(OUT)             // stw   8,0(5)