[PATCH v3] arm64/crypto: Accelerated CRC T10 DIF computation

YueHaibing yuehaibing at huawei.com
Tue Nov 22 02:14:55 PST 2016


This is the ARM64 CRC T10 DIF transform accelerated with the ARMv8
NEON instruction.The config CRYPTO_CRCT10DIF_NEON should be turned
on to enable the feature.The crc_t10dif crypto library function will
use this faster algorithm when crct10dif_neon module is loaded.

Tcrypt benchmark results:

HIP06  (mode=320 sec=2)

The ratio of bytes/sec crct10dif-neon Vs. crct10dif-generic:

                        TEST                              neon          generic         ratio
  16 byte blocks,   16 bytes per update,   1 updates    214506112       171095400       1.25
  64 byte blocks,   16 bytes per update,   4 updates    139385312       119036352       1.17
  64 byte blocks,   64 bytes per update,   1 updates    671523712       198945344       3.38
 256 byte blocks,   16 bytes per update,  16 updates    157674880       125146752       1.26
 256 byte blocks,   64 bytes per update,   4 updates    491888128       175764096       2.80
 256 byte blocks,  256 bytes per update,   1 updates    2123298176      206995200       10.26
1024 byte blocks,   16 bytes per update,  64 updates    161243136       126460416       1.28
1024 byte blocks,  256 bytes per update,   4 updates    1643020800      200027136       8.21
1024 byte blocks, 1024 bytes per update,   1 updates    4238239232      209106432       20.27
2048 byte blocks,   16 bytes per update, 128 updates    162079744       126953472       1.28
2048 byte blocks,  256 bytes per update,   8 updates    1693587456      200867840       8.43
2048 byte blocks, 1024 bytes per update,   2 updates    3424323584      206330880       16.60
2048 byte blocks, 2048 bytes per update,   1 updates    5228207104      208620544       25.06
4096 byte blocks,   16 bytes per update, 256 updates    162304000       126894080       1.28
4096 byte blocks,  256 bytes per update,  16 updates    1731862528      201197568       8.61
4096 byte blocks, 1024 bytes per update,   4 updates    3668625408      207003648       17.72
4096 byte blocks, 4096 bytes per update,   1 updates    5551239168      209127424       26.54
8192 byte blocks,   16 bytes per update, 512 updates    162779136       126984192       1.28
8192 byte blocks,  256 bytes per update,  32 updates    1753702400      201420800       8.71
8192 byte blocks, 1024 bytes per update,   8 updates    3760918528      207351808       18.14
8192 byte blocks, 4096 bytes per update,   2 updates    5483655168      208928768       26.25
8192 byte blocks, 8192 bytes per update,   1 updates    5623377920      209108992       26.89

Signed-off-by: YueHaibing <yuehaibing at huawei.com>
Signed-off-by: YangShengkai <yangshengkai at huawei.com>
Signed-off-by: Ding Tianhong <dingtianhong at huawei.com>
Signed-off-by: Hanjun Guo <hanjun.guo at linaro.org>

---
 arch/arm64/crypto/Kconfig                 |   5 +
 arch/arm64/crypto/Makefile                |   4 +
 arch/arm64/crypto/crct10dif-neon-asm_64.S | 751 ++++++++++++++++++++++++++++++
 arch/arm64/crypto/crct10dif-neon_glue.c   | 115 +++++
 4 files changed, 875 insertions(+)
 create mode 100644 arch/arm64/crypto/crct10dif-neon-asm_64.S
 create mode 100644 arch/arm64/crypto/crct10dif-neon_glue.c

diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 2cf32e9..2e450bf 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -23,6 +23,11 @@ config CRYPTO_GHASH_ARM64_CE
 	depends on ARM64 && KERNEL_MODE_NEON
 	select CRYPTO_HASH
 
+config CRYPTO_CRCT10DIF_NEON
+	tristate "CRCT10DIF hardware acceleration using NEON instructions"
+	depends on ARM64 && KERNEL_MODE_NEON
+	select CRYPTO_HASH
+
 config CRYPTO_AES_ARM64_CE
 	tristate "AES core cipher using ARMv8 Crypto Extensions"
 	depends on ARM64 && KERNEL_MODE_NEON
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index abb79b3..6c9ff2c 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -29,6 +29,10 @@ aes-ce-blk-y := aes-glue-ce.o aes-ce.o
 obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o
 aes-neon-blk-y := aes-glue-neon.o aes-neon.o
 
+obj-$(CONFIG_CRYPTO_CRCT10DIF_NEON) += crct10dif-neon.o
+crct10dif-neon-y := crct10dif-neon-asm_64.o crct10dif-neon_glue.o
+AFLAGS_crct10dif-neon-asm_64.o	:= -march=armv8-a+crypto
+
 AFLAGS_aes-ce.o		:= -DINTERLEAVE=4
 AFLAGS_aes-neon.o	:= -DINTERLEAVE=4
 
diff --git a/arch/arm64/crypto/crct10dif-neon-asm_64.S b/arch/arm64/crypto/crct10dif-neon-asm_64.S
new file mode 100644
index 0000000..2ae3033
--- /dev/null
+++ b/arch/arm64/crypto/crct10dif-neon-asm_64.S
@@ -0,0 +1,751 @@
+/*
+ * Copyright (c) 2016-2017 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+.global crc_t10dif_neon
+.text
+
+/* X0 is initial CRC value
+ * X1 is data buffer
+ * X2 is the length of buffer
+ * X3 is the backup buffer(for extend)
+ * X4 for other extend parameter(for extend)
+ * Q0, Q1, Q2, Q3 maybe as parameter for other functions,
+ * the value of Q0, Q1, Q2, Q3 maybe modified.
+ *
+ * suggestion:
+ * 1. dont use general purpose register for calculation
+ * 2. set data endianness outside of the kernel
+ * 3. use ext as shifting around
+ * 4. dont use LD3/LD4, ST3/ST4
+ */
+
+crc_t10dif_neon:
+	/* push the register to stack that CRC16 will use */
+	STP		X5, X6, [sp, #-0x10]!
+	STP		X7, X8, [sp, #-0x10]!
+	STP		X9, X10, [sp, #-0x10]!
+	STP		X11, X12, [sp, #-0x10]!
+	STP		X13, X14, [sp, #-0x10]!
+	STP		Q10, Q11, [sp, #-0x20]!
+	STP		Q12, Q13, [sp, #-0x20]!
+	STP		Q4, Q5, [sp, #-0x20]!
+	STP		Q6, Q7, [sp, #-0x20]!
+	STP		Q8, Q9, [sp, #-0x20]!
+	STP		Q14, Q15, [sp, #-0x20]!
+	STP		Q16, Q17, [sp, #-0x20]!
+	STP		Q18, Q19, [sp, #-0x20]!
+
+	SUB		sp,sp,#0x20
+
+	MOV		X11, #0		// PUSH STACK FLAG
+
+	CMP		X2, #0x80
+	B.LT		2f		// _less_than_128, <128
+
+	/* V10/V11/V12/V13	is 128bit.
+	 * we get data 512bit( by cacheline ) each time
+	 */
+	LDP		Q10, Q11, [X1], #0x20
+	LDP		Q12, Q13, [X1], #0x20
+
+	/* move the initial value to V6 register */
+	LSL		X0, X0, #48
+	EOR		V6.16B, V6.16B, V6.16B
+	MOV		V6.D[1], X0
+
+	/* big-little end change. because the data in memory is little-end,
+	 * we deal the data for bigend
+	 */
+
+	REV64		V10.16B, V10.16B
+	REV64		V11.16B, V11.16B
+	REV64		V12.16B, V12.16B
+	REV64		V13.16B, V13.16B
+	EXT     	V10.16B, V10.16B, V10.16B, #8
+	EXT     	V11.16B, V11.16B, V11.16B, #8
+	EXT     	V12.16B, V12.16B, V12.16B, #8
+	EXT     	V13.16B, V13.16B, V13.16B, #8
+
+	EOR		V10.16B, V10.16B, V6.16B
+
+	SUB		X2, X2, #0x80
+	ADD     	X5, X1, #0x20
+
+	/* deal data when the size of buffer bigger than 128 bytes */
+	/* _fold_64_B_loop */
+	LDR		Q6,=0xe658000000000000044c000000000000
+1:
+
+	LDP		Q16, Q17, [X1] ,#0x40
+	LDP		Q18, Q19, [X5], #0x40
+
+	/* carry-less multiply.
+	 * V10 high-64bits carry-less multiply
+	 * V6 high-64bits(PMULL2)
+	 * V11 low-64bits carry-less multiply V6 low-64bits(PMULL)
+	 */
+
+	PMULL2		V4.1Q, V10.2D, V6.2D
+	PMULL		V10.1Q, V10.1D, V6.1D
+	PMULL2		V5.1Q, V11.2D, V6.2D
+	PMULL		V11.1Q, V11.1D, V6.1D
+
+	REV64		V16.16B, V16.16B
+	REV64		V17.16B, V17.16B
+	REV64		V18.16B, V18.16B
+	REV64		V19.16B, V19.16B
+
+	PMULL2		V14.1Q, V12.2D, V6.2D
+	PMULL		V12.1Q, V12.1D, V6.1D
+	PMULL2		V15.1Q, V13.2D, V6.2D
+	PMULL		V13.1Q, V13.1D, V6.1D
+
+	EXT     	V16.16B, V16.16B, V16.16B, #8
+	EOR		V10.16B, V10.16B, V4.16B
+
+	EXT     	V17.16B, V17.16B, V17.16B, #8
+	EOR		V11.16B, V11.16B, V5.16B
+
+	EXT     	V18.16B, V18.16B, V18.16B, #8
+	EOR		V12.16B, V12.16B, V14.16B
+
+	EXT     	V19.16B, V19.16B, V19.16B, #8
+	EOR		V13.16B, V13.16B, V15.16B
+
+	SUB		X2, X2, #0x40
+
+
+	EOR		V10.16B, V10.16B, V16.16B
+	EOR		V11.16B, V11.16B, V17.16B
+
+	EOR		V12.16B, V12.16B, V18.16B
+	EOR		V13.16B, V13.16B, V19.16B
+
+	CMP		X2, #0x0
+	B.GE		1b			// >=0
+
+	LDR		Q6, =0x06df0000000000002d56000000000000
+	MOV		V4.16B, V10.16B
+	/* V10 carry-less 0x06df000000000000([127:64]*[127:64]) */
+	PMULL		V4.1Q, V4.1D, V6.1D	//switch PMULL & PMULL2 order
+	PMULL2		V10.1Q, V10.2D, V6.2D
+	EOR		V11.16B, V11.16B, V4.16B
+	EOR		V11.16B, V11.16B, V10.16B
+
+	MOV		V4.16B, V11.16B
+	PMULL		V4.1Q, V4.1D, V6.1D	//switch PMULL & PMULL2 order
+	PMULL2		V11.1Q, V11.2D, V6.2D
+	EOR		V12.16B, V12.16B, V4.16B
+	EOR		V12.16B, V12.16B, V11.16B
+
+	MOV		V4.16B, V12.16B
+	PMULL		V4.1Q, V4.1D, V6.1D	//switch PMULL & PMULL2 order
+	PMULL2		V12.1Q, V12.2D, V6.2D
+	EOR		V13.16B, V13.16B, V4.16B
+	EOR		V13.16B, V13.16B, V12.16B
+
+	ADD		X2, X2, #48
+	CMP		X2, #0x0
+	B.LT		3f			// _final_reduction_for_128, <0
+
+	/* _16B_reduction_loop */
+4:
+	/* unrelated load as early as possible*/
+	LDR		Q10, [X1], #0x10
+
+	MOV		V4.16B, V13.16B
+	PMULL2		V13.1Q, V13.2D, V6.2D
+	PMULL		V4.1Q, V4.1D, V6.1D
+	EOR		V13.16B, V13.16B, V4.16B
+
+	REV64		V10.16B, V10.16B
+	EXT     	V10.16B, V10.16B, V10.16B, #8
+
+	EOR		V13.16B, V13.16B, V10.16B
+
+	SUB		X2, X2, #0x10
+	CMP		X2, #0x0
+	B.GE		4b			// _16B_reduction_loop, >=0
+
+	/*  _final_reduction_for_128 */
+3:	ADD		X2, X2, #0x10
+	CMP		X2, #0x0
+	B.EQ		5f			// _128_done, ==0
+
+	/* _get_last_two_xmms */
+6:	MOV		V12.16B, V13.16B
+	SUB		X1, X1, #0x10
+	ADD		X1, X1, X2
+	LDR		Q11, [X1], #0x10
+	REV64		V11.16B, V11.16B
+	EXT		V11.16B, V11.16B, V11.16B, #8
+
+	CMP		X2, #8
+	B.EQ		50f
+	B.LT		51f
+	B.GT		52f
+
+50:
+	/* dont use X register as temp one */
+	FMOV		D14, D12
+	MOVI		D12, #0
+	MOV		V12.D[1],V14.D[0]
+	B		53f
+51:
+	MOV		X9, #64
+	LSL     	X13, X2, #3		// <<3 equal x8
+	SUB		X9, X9, X13
+	MOV		X5, V12.D[0]		// low 64-bit
+	MOV		X6, V12.D[1]		// high 64-bit
+	LSR		X10, X5, X9		// high bit of low 64-bit
+	LSL		X7, X5, X13
+	LSL		X8, X6, X13
+	ORR		X8, X8, X10		// combination of high 64-bit
+	MOV		V12.D[1], X8
+	MOV		V12.D[0], X7
+
+	B		53f
+52:
+	LSL		X13, X2, #3		// <<3 equal x8
+	SUB		X13, X13, #64
+
+	DUP		V18.2D, X13
+	FMOV		D16, D12
+	USHL		D16, D16, D18
+	EXT		V12.16B, V16.16B, V16.16B, #8
+
+53:
+	MOVI		D14, #0			//add one zero constant
+
+	CMP		X2, #0
+	B.EQ	30f
+	CMP		X2, #1
+	B.EQ	31f
+	CMP		X2, #2
+	B.EQ	32f
+	CMP		X2, #3
+	B.EQ	33f
+	CMP		X2, #4
+	B.EQ	34f
+	CMP		X2, #5
+	B.EQ	35f
+	CMP		X2, #6
+	B.EQ	36f
+	CMP		X2, #7
+	B.EQ	37f
+	CMP		X2, #8
+	B.EQ	38f
+	CMP		X2, #9
+	B.EQ	39f
+	CMP		X2, #10
+	B.EQ	40f
+	CMP		X2, #11
+	B.EQ	41f
+	CMP		X2, #12
+	B.EQ	42f
+	CMP		X2, #13
+	B.EQ	43f
+	CMP		X2, #14
+	B.EQ	44f
+	CMP		X2, #15
+	B.EQ	45f
+
+	// >> 128bit
+30:
+	EOR		V13.16B, V13.16B, V13.16B
+	EOR		V8.16B, V8.16B, V8.16B
+	LDR		Q9,=0xffffffffffffffffffffffffffffffff
+	B		46f
+
+	// >> 120bit
+31:
+	USHR		V13.2D, V13.2D, #56
+	EXT		V13.16B, V13.16B, V14.16B, #8
+	LDR		Q8,=0xff
+	LDR		Q9,=0xffffffffffffffffffffffffffffff00
+	B		46f
+
+	// >> 112bit
+32:
+	USHR		 V13.2D, V13.2D, #48
+	EXT		V13.16B, V13.16B, V14.16B, #8
+	LDR		Q8,=0xffff
+	LDR		Q9,=0xffffffffffffffffffffffffffff0000
+	B		46f
+
+	// >> 104bit
+33:
+	USHR		V13.2D, V13.2D, #40
+	EXT		V13.16B, V13.16B, V14.16B, #8
+	LDR		Q8,=0xffffff
+	LDR		Q9,=0xffffffffffffffffffffffffff000000
+	B		46f
+
+	// >> 96bit
+34:
+	USHR		V13.2D, V13.2D, #32
+	EXT		V13.16B, V13.16B, V14.16B, #8
+	LDR		Q8,=0xffffffff
+	LDR		Q9,=0xffffffffffffffffffffffff00000000
+	B		46f
+
+	// >> 88bit
+35:
+	USHR		V13.2D, V13.2D, #24
+	EXT		V13.16B, V13.16B, V14.16B, #8
+	LDR		Q8,=0xffffffffff
+	LDR		Q9,=0xffffffffffffffffffffff0000000000
+	B		46f
+
+	// >> 80bit
+36:
+	USHR		V13.2D, V13.2D, #16
+	EXT		V13.16B, V13.16B, V14.16B, #8
+	LDR		Q8,=0xffffffffffff
+	LDR		Q9,=0xffffffffffffffffffff000000000000
+	B		46f
+
+	// >> 72bit
+37:
+	USHR		V13.2D, V13.2D, #8
+	EXT		V13.16B, V13.16B, V14.16B, #8
+	LDR		Q8,=0xffffffffffffff
+	LDR		Q9,=0xffffffffffffffffff00000000000000
+	B		46f
+
+	// >> 64bit
+38:
+	EXT		 V13.16B, V13.16B, V14.16B, #8
+	LDR		Q8,=0xffffffffffffffff
+	LDR		Q9,=0xffffffffffffffff0000000000000000
+	B		46f
+
+	// >> 56bit
+39:
+	EXT		V13.16B, V13.16B, V13.16B, #7
+	MOV		V13.S[3], V14.S[0]
+	MOV		V13.H[5], V14.H[0]
+	MOV		V13.B[9], V14.B[0]
+
+	LDR		Q8,=0xffffffffffffffffff
+	LDR		Q9,=0xffffffffffffff000000000000000000
+	B		46f
+
+	// >> 48bit
+40:
+	EXT		V13.16B, V13.16B, V13.16B, #6
+	MOV		V13.S[3], V14.S[0]
+	MOV		V13.H[5], V14.H[0]
+
+	LDR		Q8,=0xffffffffffffffffffff
+	LDR		Q9,=0xffffffffffff00000000000000000000
+	B		46f
+
+	// >> 40bit
+41:
+	EXT		V13.16B, V13.16B, V13.16B, #5
+	MOV		V13.S[3], V14.S[0]
+	MOV		V13.B[11], V14.B[0]
+
+	LDR		Q8,=0xffffffffffffffffffffff
+	LDR		Q9,=0xffffffffff0000000000000000000000
+	B		46f
+
+	// >> 32bit
+42:
+	EXT		V13.16B, V13.16B, V13.16B, #4
+	MOV		V13.S[3], V14.S[0]
+
+	LDR		Q8,=0xffffffffffffffffffffffff
+	LDR		Q9,=0xffffffff000000000000000000000000
+	B		46f
+
+	// >> 24bit
+43:
+	EXT		V13.16B, V13.16B, V13.16B, #3
+	MOV		V13.H[7], V14.H[0]
+	MOV		V13.B[13], V14.B[0]
+
+	LDR		Q8,=0xffffffffffffffffffffffffff
+	LDR		Q9,=0xffffff00000000000000000000000000
+	B		46f
+
+	// >> 16bit
+44:
+	EXT		V13.16B, V13.16B, V13.16B, #2
+	MOV		V13.H[7], V14.H[0]
+
+	LDR		Q8,=0xffffffffffffffffffffffffffff
+	LDR		Q9,=0xffff0000000000000000000000000000
+	B		46f
+
+	// >> 8bit
+45:
+	EXT		 V13.16B, V13.16B, V13.16B, #1
+	MOV		V13.B[15], V14.B[0]
+
+	LDR		Q8,=0xffffffffffffffffffffffffffffff
+	LDR		Q9,=0xff000000000000000000000000000000
+
+	// backup V12 first
+	// pblendvb	xmm1, xmm2
+46:
+	AND		V12.16B, V12.16B, V9.16B
+	AND		V11.16B, V11.16B, V8.16B
+	ORR		V11.16B, V11.16B, V12.16B
+
+	MOV		V12.16B, V11.16B
+	MOV		V4.16B, V13.16B
+	PMULL2		V13.1Q, V13.2D, V6.2D
+	PMULL		V4.1Q, V4.1D, V6.1D
+	EOR		V13.16B, V13.16B, V4.16B
+	EOR		V13.16B, V13.16B, V12.16B
+
+	/* _128_done. we change the Q6 D[0] and D[1] */
+5:	LDR		Q6, =0x2d560000000000001368000000000000
+	MOVI		D14, #0
+	MOV		V10.16B, V13.16B
+	PMULL2		V13.1Q, V13.2D, V6.2D
+
+	MOV		V10.D[1], V10.D[0]
+	MOV		V10.D[0], V14.D[0]    //set zero
+
+	EOR		V13.16B, V13.16B, V10.16B
+
+	MOV		V10.16B, V13.16B
+	LDR		Q7, =0x00000000FFFFFFFFFFFFFFFFFFFFFFFF
+	AND		V10.16B, V10.16B, V7.16B
+
+	MOV		S13, V13.S[3]
+
+	PMULL		V13.1Q, V13.1D, V6.1D
+	EOR		V13.16B, V13.16B, V10.16B
+
+	/* _barrett */
+7:	LDR		Q6, =0x00000001f65a57f8000000018bb70000
+	MOVI    	D14, #0
+	MOV		V10.16B, V13.16B
+	PMULL2		V13.1Q, V13.2D, V6.2D
+
+	EXT		V13.16B, V13.16B, V13.16B, #12
+	MOV		V13.S[0], V14.S[0]
+
+	EXT		V6.16B, V6.16B, V6.16B, #8
+	PMULL2		V13.1Q, V13.2D, V6.2D
+
+	EXT		V13.16B, V13.16B, V13.16B, #12
+	MOV		V13.S[0], V14.S[0]
+
+	EOR		V13.16B, V13.16B, V10.16B
+	MOV		X0, V13.D[0]
+
+	/* _cleanup */
+8:	MOV		X14, #48
+	LSR		X0, X0, X14
+99:
+	ADD		sp, sp, #0x20
+
+	LDP		Q18, Q19, [sp], #0x20
+	LDP		Q16, Q17, [sp], #0x20
+	LDP		Q14, Q15, [sp], #0x20
+
+	LDP		Q8, Q9, [sp], #0x20
+	LDP		Q6, Q7, [sp], #0x20
+	LDP		Q4, Q5, [sp], #0x20
+	LDP		Q12, Q13, [sp], #0x20
+	LDP		Q10, Q11, [sp], #0x20
+	LDP		X13, X14, [sp], #0x10
+	LDP		X11, X12, [sp], #0x10
+	LDP		X9, X10, [sp], #0x10
+	LDP		X7, X8, [sp], #0x10
+	LDP		X5, X6, [sp], #0x10
+
+	RET
+
+	/* _less_than_128 */
+2:	CMP		X2, #32
+	B.LT		9f				// _less_than_32
+	LDR		Q6, =0x06df0000000000002d56000000000000
+
+	LSL		X0, X0, #48
+	LDR		Q10, =0x0
+	MOV		V10.D[1], X0
+	LDR		Q13, [X1], #0x10
+	REV64		V13.16B, V13.16B
+	EXT     	V13.16B, V13.16B, V13.16B, #8
+
+	EOR		V13.16B, V13.16B, V10.16B
+
+	SUB		X2, X2, #32
+	B		4b
+
+	/* _less_than_32 */
+9:	CMP		X2, #0
+	B.EQ		99b			// _cleanup
+	LSL		X0, X0, #48
+	LDR		Q10,=0x0
+	MOV		V10.D[1], X0
+
+	CMP		X2, #16
+	B.EQ		10f			// _exact_16_left
+	B.LE		11f			// _less_than_16_left
+	LDR		Q13, [X1], #0x10
+
+	REV64		V13.16B, V13.16B
+	EXT		V13.16B, V13.16B, V13.16B, #8
+
+	EOR		V13.16B, V13.16B, V10.16B
+	SUB		X2, X2, #16
+	LDR		Q6, =0x06df0000000000002d56000000000000
+	B		6b			// _get_last_two_xmms
+
+	/*  _less_than_16_left */
+11:	CMP		X2, #4
+	B.LT		13f			// _only_less_than_4
+
+	/* backup the length of data, we used in _less_than_2_left */
+	MOV		X8, X2
+	CMP		X2, #8
+	B.LT		14f			// _less_than_8_left
+
+	LDR		X14, [X1], #8
+	/* push the data to stack, we backup the data to V10 */
+	STR		X14, [sp, #0]
+	SUB		X2, X2, #8
+	ADD		X11, X11, #8
+
+	/* _less_than_8_left */
+14:	CMP		X2, #4
+	B.LT		15f			// _less_than_4_left
+
+	/* get 32bit data */
+	LDR		W5, [X1], #4
+
+	/* push the data to stack */
+	STR		W5, [sp, X11]
+	SUB		X2, X2, #4
+	ADD		X11, X11, #4
+
+	/* _less_than_4_left */
+15:	CMP		X2, #2
+	B.LT		16f			// _less_than_2_left
+
+	/* get 16bits data */
+	LDRH		W6, [X1], #2
+
+	/* push the data to stack */
+	STRH		W6, [sp, X11]
+	SUB		X2, X2, #2
+	ADD		X11, X11, #2
+
+	/* _less_than_2_left */
+16:
+	/* get 8bits data */
+	LDRB		W7, [X1], #1
+	STRB		W7, [sp, X11]
+	ADD		X11, X11, #1
+
+	/* POP data from stack, store to V13 */
+	LDR		Q13, [sp]
+	MOVI    	D14, #0
+	REV64		V13.16B, V13.16B
+	MOV		V8.16B, V13.16B
+	MOV		V13.D[1], V8.D[0]
+	MOV		V13.D[0], V8.D[1]
+
+	EOR		V13.16B, V13.16B, V10.16B
+	CMP		X8, #15
+	B.EQ	80f
+	CMP		X8, #14
+	B.EQ	81f
+	CMP		X8, #13
+	B.EQ	82f
+	CMP		X8, #12
+	B.EQ	83f
+	CMP		X8, #11
+	B.EQ	84f
+	CMP		X8, #10
+	B.EQ	85f
+	CMP		X8, #9
+	B.EQ	86f
+	CMP		X8, #8
+	B.EQ	87f
+	CMP		X8, #7
+	B.EQ	88f
+	CMP		X8, #6
+	B.EQ	89f
+	CMP		X8, #5
+	B.EQ	90f
+	CMP		X8, #4
+	B.EQ	91f
+	CMP		X8, #3
+	B.EQ	92f
+	CMP		X8, #2
+	B.EQ	93f
+	CMP		X8, #1
+	B.EQ	94f
+	CMP		X8, #0
+	B.EQ	95f
+
+80:
+	EXT		V13.16B, V13.16B, V13.16B, #1
+	MOV		V13.B[15], V14.B[0]
+	B		5b
+
+81:
+	EXT		V13.16B, V13.16B, V13.16B, #2
+	MOV		 V13.H[7], V14.H[0]
+	B		5b
+
+82:
+	EXT		V13.16B, V13.16B, V13.16B, #3
+	MOV		V13.H[7], V14.H[0]
+	MOV		V13.B[13], V14.B[0]
+	B		5b
+83:
+
+	EXT		V13.16B, V13.16B, V13.16B, #4
+	MOV		V13.S[3], V14.S[0]
+	B		5b
+
+84:
+	EXT		V13.16B, V13.16B, V13.16B, #5
+	MOV		V13.S[3], V14.S[0]
+	MOV		V13.B[11], V14.B[0]
+	B		5b
+
+85:
+	EXT		V13.16B, V13.16B, V13.16B, #6
+	MOV		V13.S[3], V14.S[0]
+	MOV		V13.H[5], V14.H[0]
+	B		5b
+
+86:
+	EXT		V13.16B, V13.16B, V13.16B, #7
+	MOV		V13.S[3], V14.S[0]
+	MOV		V13.H[5], V14.H[0]
+	MOV		V13.B[9], V14.B[0]
+	B		5b
+
+87:
+	MOV		V13.D[0], V13.D[1]
+	MOV		V13.D[1], V14.D[0]
+	B		5b
+
+88:
+	EXT		V13.16B, V13.16B, V13.16B, #9
+	MOV		V13.D[1], V14.D[0]
+	MOV		V13.B[7], V14.B[0]
+	B		5b
+
+89:
+	EXT		V13.16B, V13.16B, V13.16B, #10
+	MOV		V13.D[1], V14.D[0]
+	MOV		 V13.H[3], V14.H[0]
+	B		5b
+
+90:
+	EXT		V13.16B, V13.16B, V13.16B, #11
+	MOV		V13.D[1], V14.D[0]
+	MOV		V13.H[3], V14.H[0]
+	MOV		V13.B[5], V14.B[0]
+	B		5b
+
+91:
+	MOV		V13.S[0], V13.S[3]
+	MOV		V13.D[1], V14.D[0]
+	MOV		V13.S[1], V14.S[0]
+	B		5b
+
+92:
+	EXT		V13.16B, V13.16B, V13.16B, #13
+	MOV		V13.D[1], V14.D[0]
+	MOV		V13.S[1], V14.S[0]
+	MOV		V13.B[3], V14.B[0]
+	B		5b
+
+93:
+	MOV		V15.H[0], V13.H[7]
+	MOV		V13.16B, V14.16B
+	MOV		V13.H[0], V15.H[0]
+	B		5b
+
+94:
+	MOV		V15.B[0], V13.B[15]
+	MOV		V13.16B, V14.16B
+	MOV		V13.B[0], V15.B[0]
+	B		5b
+
+95:
+	LDR		Q13,=0x0
+	B		5b				// _128_done
+
+	/* _exact_16_left */
+10:
+	LD1		{ V13.2D }, [X1], #0x10
+
+	REV64		V13.16B, V13.16B
+	EXT		V13.16B, V13.16B, V13.16B, #8
+	EOR		V13.16B, V13.16B, V10.16B
+	B		5b				// _128_done
+
+	/* _only_less_than_4 */
+13:	CMP		X2, #3
+	MOVI		D14, #0
+	B.LT		17f				//_only_less_than_3
+
+	LDR		S13, [X1], #4
+	MOV		 V13.B[15], V13.B[0]
+	MOV		V13.B[14], V13.B[1]
+	MOV		V13.B[13], V13.B[2]
+	MOV		V13.S[0], V13.S[1]
+
+	EOR		V13.16B, V13.16B, V10.16B
+
+	EXT		V13.16B, V13.16B, V13.16B, #5
+
+	MOV		V13.S[3], V14.S[0]
+	MOV		V13.B[11], V14.B[0]
+
+	B		7b				// _barrett
+	/* _only_less_than_3 */
+17:
+	CMP		X2, #2
+	B.LT		18f				// _only_less_than_2
+
+	LDR		H13, [X1], #2
+	MOV		V13.B[15], V13.B[0]
+	MOV		V13.B[14], V13.B[1]
+	MOV		V13.H[0], V13.H[1]
+
+	EOR		V13.16B, V13.16B, V10.16B
+
+	EXT		V13.16B, V13.16B, V13.16B, #6
+	MOV		V13.S[3], V14.S[0]
+	MOV		V13.H[5], V14.H[0]
+
+	B		7b				// _barrett
+
+	/* _only_less_than_2 */
+18:
+	LDRB		W7, [X1], #1
+	LDR		Q13, = 0x0
+	MOV		V13.B[15], W7
+
+	EOR		V13.16B, V13.16B, V10.16B
+
+	EXT		V13.16B, V13.16B, V13.16B, #7
+	MOV		V13.S[3], V14.S[0]
+	MOV		V13.H[5], V14.H[0]
+	MOV		V13.B[9], V14.B[0]
+
+	B		7b				// _barrett
diff --git a/arch/arm64/crypto/crct10dif-neon_glue.c b/arch/arm64/crypto/crct10dif-neon_glue.c
new file mode 100644
index 0000000..a6d8c5c
--- /dev/null
+++ b/arch/arm64/crypto/crct10dif-neon_glue.c
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2016-2017 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/crc-t10dif.h>
+#include <crypto/internal/hash.h>
+#include <linux/init.h>
+#include <linux/string.h>
+#include <linux/kernel.h>
+
+asmlinkage __u16 crc_t10dif_neon(__u16 crc, const unsigned char *buf,
+				size_t len);
+
+struct chksum_desc_ctx {
+	__u16 crc;
+};
+
+/*
+ * Steps through buffer one byte at at time, calculates reflected
+ * crc using table.
+ */
+
+static int chksum_init(struct shash_desc *desc)
+{
+	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+	ctx->crc = 0;
+
+	return 0;
+}
+
+static int chksum_update(struct shash_desc *desc, const u8 *data,
+			 unsigned int length)
+{
+	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+	ctx->crc = crc_t10dif_neon(ctx->crc, data, length);
+	return 0;
+}
+
+static int chksum_final(struct shash_desc *desc, u8 *out)
+{
+	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+	*(__u16 *)out = ctx->crc;
+	return 0;
+}
+
+static int __chksum_finup(__u16 *crcp, const u8 *data, unsigned int len,
+			u8 *out)
+{
+	*(__u16 *)out = crc_t10dif_neon(*crcp, data, len);
+	return 0;
+}
+
+static int chksum_finup(struct shash_desc *desc, const u8 *data,
+			unsigned int len, u8 *out)
+{
+	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+	return __chksum_finup(&ctx->crc, data, len, out);
+}
+
+static int chksum_digest(struct shash_desc *desc, const u8 *data,
+			 unsigned int length, u8 *out)
+{
+	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+	return __chksum_finup(&ctx->crc, data, length, out);
+}
+
+static struct shash_alg alg = {
+	.digestsize		=	CRC_T10DIF_DIGEST_SIZE,
+	.init		=	chksum_init,
+	.update		=	chksum_update,
+	.final		=	chksum_final,
+	.finup		=	chksum_finup,
+	.digest		=	chksum_digest,
+	.descsize		=	sizeof(struct chksum_desc_ctx),
+	.base			=	{
+		.cra_name		=	"crct10dif",
+		.cra_driver_name	=	"crct10dif-neon",
+		.cra_priority		=	200,
+		.cra_blocksize		=	CRC_T10DIF_BLOCK_SIZE,
+		.cra_module		=	THIS_MODULE,
+	}
+};
+
+static int __init crct10dif_arm64_mod_init(void)
+{
+	return crypto_register_shash(&alg);
+}
+
+static void __exit crct10dif_arm64_mod_fini(void)
+{
+	crypto_unregister_shash(&alg);
+}
+
+module_init(crct10dif_arm64_mod_init);
+module_exit(crct10dif_arm64_mod_fini);
+
+MODULE_AUTHOR("YueHaibing <yuehaibing at huawei.com>");
+MODULE_DESCRIPTION("T10 DIF CRC calculation accelerated with ARM64 NEON instruction.");
+MODULE_LICENSE("GPL");
+
+MODULE_ALIAS_CRYPTO("crct10dif");
+MODULE_ALIAS_CRYPTO("crct10dif-neon");
-- 
2.7.0





More information about the linux-arm-kernel mailing list