From 501b389efc6213eda80eeed2253d3d7f931f3db5 Mon Sep 17 00:00:00 2001 From: Xiaolin Zhao Date: Mon, 3 Jun 2024 15:49:23 +0800 Subject: [PATCH] runtime: optimize the function memclrNoHeapPointers on loong64 MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit The relevant performance improved by 54.61%. benchmark: goos: linux goarch: loong64 pkg: runtime cpu: Loongson-3A6000 @ 2500.00MHz │ old │ new │ │ sec/op │ sec/op vs base │ Memclr/5 4.803n ± 0% 2.801n ± 0% -41.68% (p=0.000 n=20) Memclr/16 4.803n ± 0% 3.202n ± 0% -33.33% (p=0.000 n=20) Memclr/64 9.605n ± 0% 5.061n ± 1% -47.30% (p=0.000 n=20) Memclr/256 29.22n ± 0% 10.24n ± 0% -64.96% (p=0.000 n=20) Memclr/4096 413.4n ± 0% 106.9n ± 0% -74.14% (p=0.000 n=20) Memclr/65536 6.566µ ± 0% 1.673µ ± 0% -74.52% (p=0.000 n=20) Memclr/1M 104.95µ ± 0% 52.51µ ± 0% -49.97% (p=0.000 n=20) Memclr/4M 419.8µ ± 0% 209.9µ ± 0% -49.99% (p=0.000 n=20) Memclr/8M 839.6µ ± 0% 419.9µ ± 0% -49.98% (p=0.000 n=20) Memclr/16M 1687.6µ ± 0% 845.3µ ± 0% -49.91% (p=0.000 n=20) Memclr/64M 6.725m ± 0% 3.389m ± 0% -49.61% (p=0.000 n=20) MemclrUnaligned/0_5 6.003n ± 0% 4.581n ± 0% -23.69% (p=0.000 n=20) MemclrUnaligned/0_16 6.005n ± 0% 5.084n ± 0% -15.33% (p=0.000 n=20) MemclrUnaligned/0_64 10.810n ± 0% 6.229n ± 0% -42.38% (p=0.000 n=20) MemclrUnaligned/0_256 30.43n ± 0% 10.68n ± 0% -64.90% (p=0.000 n=20) MemclrUnaligned/0_4096 414.8n ± 0% 107.1n ± 0% -74.18% (p=0.000 n=20) MemclrUnaligned/0_65536 6.566µ ± 0% 1.700µ ± 0% -74.11% (p=0.000 n=20) MemclrUnaligned/1_5 6.003n ± 0% 4.582n ± 0% -23.67% (p=0.000 n=20) MemclrUnaligned/1_16 11.610n ± 0% 5.080n ± 0% -56.24% (p=0.000 n=20) MemclrUnaligned/1_64 16.810n ± 0% 7.370n ± 0% -56.16% (p=0.000 n=20) MemclrUnaligned/1_256 36.42n ± 0% 12.95n ± 0% -64.44% (p=0.000 n=20) MemclrUnaligned/1_4096 420.6n ± 0% 114.6n ± 0% -72.75% (p=0.000 n=20) MemclrUnaligned/1_65536 6.573µ ± 0% 1.708µ ± 0% -74.01% (p=0.000 n=20) MemclrUnaligned/4_5 6.003n ± 0% 4.582n ± 0% -23.67% (p=0.000 n=20) MemclrUnaligned/4_16 10.410n ± 0% 5.069n ± 0% -51.30% (p=0.000 n=20) MemclrUnaligned/4_64 15.610n ± 0% 7.372n ± 0% -52.77% (p=0.000 n=20) MemclrUnaligned/4_256 35.22n ± 0% 12.95n ± 0% -63.23% (p=0.000 n=20) MemclrUnaligned/4_4096 419.4n ± 0% 114.6n ± 0% -72.68% (p=0.000 n=20) MemclrUnaligned/4_65536 6.571µ ± 0% 1.708µ ± 0% -74.01% (p=0.000 n=20) MemclrUnaligned/7_5 6.003n ± 0% 4.581n ± 0% -23.69% (p=0.000 n=20) MemclrUnaligned/7_16 8.855n ± 0% 5.079n ± 0% -42.65% (p=0.000 n=20) MemclrUnaligned/7_64 14.010n ± 0% 7.370n ± 0% -47.39% (p=0.000 n=20) MemclrUnaligned/7_256 33.62n ± 0% 12.95n ± 0% -61.48% (p=0.000 n=20) MemclrUnaligned/7_4096 417.8n ± 0% 114.7n ± 0% -72.56% (p=0.000 n=20) MemclrUnaligned/7_65536 6.570µ ± 0% 1.708µ ± 0% -74.00% (p=0.000 n=20) MemclrUnaligned/0_1M 104.96µ ± 0% 52.51µ ± 0% -49.97% (p=0.000 n=20) MemclrUnaligned/0_4M 419.8µ ± 0% 209.9µ ± 0% -49.99% (p=0.000 n=20) MemclrUnaligned/0_8M 839.5µ ± 0% 419.8µ ± 0% -49.99% (p=0.000 n=20) MemclrUnaligned/0_16M 1687.9µ ± 0% 844.9µ ± 0% -49.94% (p=0.000 n=20) MemclrUnaligned/0_64M 6.725m ± 0% 3.382m ± 0% -49.72% (p=0.000 n=20) MemclrUnaligned/1_1M 104.97µ ± 0% 52.51µ ± 0% -49.97% (p=0.000 n=20) MemclrUnaligned/1_4M 419.8µ ± 0% 210.0µ ± 0% -49.97% (p=0.000 n=20) MemclrUnaligned/1_8M 839.5µ ± 0% 419.8µ ± 0% -50.00% (p=0.000 n=20) MemclrUnaligned/1_16M 1687.6µ ± 0% 844.2µ ± 0% -49.97% (p=0.000 n=20) MemclrUnaligned/1_64M 6.724m ± 0% 3.367m ± 0% -49.93% (p=0.000 n=20) MemclrUnaligned/4_1M 104.97µ ± 0% 52.51µ ± 0% -49.97% (p=0.000 n=20) MemclrUnaligned/4_4M 419.8µ ± 0% 210.0µ ± 0% -49.97% (p=0.000 n=20) MemclrUnaligned/4_8M 839.5µ ± 0% 419.8µ ± 0% -50.00% (p=0.000 n=20) MemclrUnaligned/4_16M 1687.5µ ± 0% 844.4µ ± 0% -49.96% (p=0.000 n=20) MemclrUnaligned/4_64M 6.725m ± 0% 3.366m ± 0% -49.95% (p=0.000 n=20) MemclrUnaligned/7_1M 104.97µ ± 0% 52.51µ ± 0% -49.97% (p=0.000 n=20) MemclrUnaligned/7_4M 419.8µ ± 0% 210.0µ ± 0% -49.97% (p=0.000 n=20) MemclrUnaligned/7_8M 839.5µ ± 0% 419.8µ ± 0% -50.00% (p=0.000 n=20) MemclrUnaligned/7_16M 1687.9µ ± 0% 844.3µ ± 0% -49.98% (p=0.000 n=20) MemclrUnaligned/7_64M 6.724m ± 0% 3.362m ± 0% -50.00% (p=0.000 n=20) geomean 4.659µ 2.114µ -54.61% Change-Id: If0174e4cd8be5e17ad146698508a966158fe83e8 Reviewed-on: https://go-review.googlesource.com/c/go/+/589539 Reviewed-by: Qiqi Huang Reviewed-by: Michael Pratt Reviewed-by: Dmitri Shuralyov LUCI-TryBot-Result: Go LUCI Reviewed-by: abner chenc Reviewed-by: Meidan Li --- src/runtime/memclr_loong64.s | 139 +++++++++++++++++++++++++++++------ 1 file changed, 117 insertions(+), 22 deletions(-) diff --git a/src/runtime/memclr_loong64.s b/src/runtime/memclr_loong64.s index 1d45e82d49..346b210c8d 100644 --- a/src/runtime/memclr_loong64.s +++ b/src/runtime/memclr_loong64.s @@ -5,36 +5,131 @@ #include "go_asm.h" #include "textflag.h" +// Register map +// +// R4: ptr +// R5: n +// R6: ptrend +// R7: tmp + +// Algorithm: +// +// 1. when count <= 64 bytes, memory alignment check is omitted. +// The handling is divided into distinct cases based on the size +// of count: clr_0, clr_1, clr_2, clr_3, clr_4, clr_5through7, +// clr_8, clr_9through16, clr_17through32, and clr_33through64. +// +// 2. when count > 64 bytes, memory alignment check is performed. +// Unaligned bytes are processed first (that is, 8-(ptr&7)), and +// then a 64-byte loop is executed to zero out memory. +// When the number of remaining bytes not cleared is n < 64 bytes, +// a tail processing is performed, invoking the corresponding case +// based on the size of n. +// +// ptr newptr ptrend +// | |<----count after correction---->| +// |<-------------count before correction---------->| +// |<--8-(ptr&7)-->| |<---64 bytes--->| +// +------------------------------------------------+ +// | Head | Body | Tail | +// +---------------+---------------+----------------+ +// newptr = ptr - (ptr & 7) + 8 +// count = count - 8 + (ptr & 7) + // func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) TEXT runtime·memclrNoHeapPointers(SB),NOSPLIT,$0-16 + BEQ R5, clr_0 ADDV R4, R5, R6 - // if less than 8 bytes, do one byte at a time - SGTU $8, R5, R8 - BNE R8, out +tail: + // <=64 bytes, clear directly, not check aligned + SGTU $2, R5, R7 + BNE R7, clr_1 + SGTU $3, R5, R7 + BNE R7, clr_2 + SGTU $4, R5, R7 + BNE R7, clr_3 + SGTU $5, R5, R7 + BNE R7, clr_4 + SGTU $8, R5, R7 + BNE R7, clr_5through7 + SGTU $9, R5, R7 + BNE R7, clr_8 + SGTU $17, R5, R7 + BNE R7, clr_9through16 + SGTU $33, R5, R7 + BNE R7, clr_17through32 + SGTU $65, R5, R7 + BNE R7, clr_33through64 - // do one byte at a time until 8-aligned - AND $7, R4, R8 - BEQ R8, words - MOVB R0, (R4) - ADDV $1, R4 - JMP -4(PC) + // n > 64 bytes, check aligned + AND $7, R4, R7 + BEQ R7, body -words: - // do 8 bytes at a time if there is room - ADDV $-7, R6, R5 +head: + MOVV R0, (R4) + SUBV R7, R4 + ADDV R7, R5 + ADDV $8, R4 // newptr = ptr + (8 - (ptr & 7)) + SUBV $8, R5 // newn = n - (8 - (ptr & 7)) + SGTU $65, R5, R7 + BNE R7, clr_33through64 - PCALIGN $16 - SGTU R5, R4, R8 - BEQ R8, out +body: MOVV R0, (R4) - ADDV $8, R4 - JMP -4(PC) + MOVV R0, 8(R4) + MOVV R0, 16(R4) + MOVV R0, 24(R4) + MOVV R0, 32(R4) + MOVV R0, 40(R4) + MOVV R0, 48(R4) + MOVV R0, 56(R4) + ADDV $-64, R5 + ADDV $64, R4 + SGTU $65, R5, R7 + BEQ R7, body + BEQ R5, clr_0 + JMP tail -out: - BEQ R4, R6, done +clr_0: + RET +clr_1: MOVB R0, (R4) - ADDV $1, R4 - JMP -3(PC) -done: + RET +clr_2: + MOVH R0, (R4) + RET +clr_3: + MOVH R0, (R4) + MOVB R0, 2(R4) + RET +clr_4: + MOVW R0, (R4) + RET +clr_5through7: + MOVW R0, (R4) + MOVW R0, -4(R6) + RET +clr_8: + MOVV R0, (R4) + RET +clr_9through16: + MOVV R0, (R4) + MOVV R0, -8(R6) + RET +clr_17through32: + MOVV R0, (R4) + MOVV R0, 8(R4) + MOVV R0, -16(R6) + MOVV R0, -8(R6) + RET +clr_33through64: + MOVV R0, (R4) + MOVV R0, 8(R4) + MOVV R0, 16(R4) + MOVV R0, 24(R4) + MOVV R0, -32(R6) + MOVV R0, -24(R6) + MOVV R0, -16(R6) + MOVV R0, -8(R6) RET -- 2.48.1