1 files changed, 0 insertions, 361 deletions
diff --git a/arch/arm64/crypto/polyval-ce-core.S b/arch/arm64/crypto/polyval-ce-core.S
deleted file mode 100644
index b5326540d2e3..000000000000
--- a/arch/arm64/crypto/polyval-ce-core.S
+++ /dev/null
@@ -1,361 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Implementation of POLYVAL using ARMv8 Crypto Extensions.
- *
- * Copyright 2021 Google LLC
- */
-/*
- * This is an efficient implementation of POLYVAL using ARMv8 Crypto Extensions
- * It works on 8 blocks at a time, by precomputing the first 8 keys powers h^8,
- * ..., h^1 in the POLYVAL finite field. This precomputation allows us to split
- * finite field multiplication into two steps.
- *
- * In the first step, we consider h^i, m_i as normal polynomials of degree less
- * than 128. We then compute p(x) = h^8m_0 + ... + h^1m_7 where multiplication
- * is simply polynomial multiplication.
- *
- * In the second step, we compute the reduction of p(x) modulo the finite field
- * modulus g(x) = x^128 + x^127 + x^126 + x^121 + 1.
- *
- * This two step process is equivalent to computing h^8m_0 + ... + h^1m_7 where
- * multiplication is finite field multiplication. The advantage is that the
- * two-step process  only requires 1 finite field reduction for every 8
- * polynomial multiplications. Further parallelism is gained by interleaving the
- * multiplications and polynomial reductions.
- */
-
-#include <linux/linkage.h>
-#define STRIDE_BLOCKS 8
-
-KEY_POWERS	.req	x0
-MSG		.req	x1
-BLOCKS_LEFT	.req	x2
-ACCUMULATOR	.req	x3
-KEY_START	.req	x10
-EXTRA_BYTES	.req	x11
-TMP	.req	x13
-
-M0	.req	v0
-M1	.req	v1
-M2	.req	v2
-M3	.req	v3
-M4	.req	v4
-M5	.req	v5
-M6	.req	v6
-M7	.req	v7
-KEY8	.req	v8
-KEY7	.req	v9
-KEY6	.req	v10
-KEY5	.req	v11
-KEY4	.req	v12
-KEY3	.req	v13
-KEY2	.req	v14
-KEY1	.req	v15
-PL	.req	v16
-PH	.req	v17
-TMP_V	.req	v18
-LO	.req	v20
-MI	.req	v21
-HI	.req	v22
-SUM	.req	v23
-GSTAR	.req	v24
-
-	.text
-
-	.arch	armv8-a+crypto
-	.align	4
-
-.Lgstar:
-	.quad	0xc200000000000000, 0xc200000000000000
-
-/*
- * Computes the product of two 128-bit polynomials in X and Y and XORs the
- * components of the 256-bit product into LO, MI, HI.
- *
- * Given:
- *  X = [X_1 : X_0]
- *  Y = [Y_1 : Y_0]
- *
- * We compute:
- *  LO += X_0 * Y_0
- *  MI += (X_0 + X_1) * (Y_0 + Y_1)
- *  HI += X_1 * Y_1
- *
- * Later, the 256-bit result can be extracted as:
- *   [HI_1 : HI_0 + HI_1 + MI_1 + LO_1 : LO_1 + HI_0 + MI_0 + LO_0 : LO_0]
- * This step is done when computing the polynomial reduction for efficiency
- * reasons.
- *
- * Karatsuba multiplication is used instead of Schoolbook multiplication because
- * it was found to be slightly faster on ARM64 CPUs.
- *
- */
-.macro karatsuba1 X Y
-	X .req \X
-	Y .req \Y
-	ext	v25.16b, X.16b, X.16b, #8
-	ext	v26.16b, Y.16b, Y.16b, #8
-	eor	v25.16b, v25.16b, X.16b
-	eor	v26.16b, v26.16b, Y.16b
-	pmull2	v28.1q, X.2d, Y.2d
-	pmull	v29.1q, X.1d, Y.1d
-	pmull	v27.1q, v25.1d, v26.1d
-	eor	HI.16b, HI.16b, v28.16b
-	eor	LO.16b, LO.16b, v29.16b
-	eor	MI.16b, MI.16b, v27.16b
-	.unreq X
-	.unreq Y
-.endm
-
-/*
- * Same as karatsuba1, except overwrites HI, LO, MI rather than XORing into
- * them.
- */
-.macro karatsuba1_store X Y
-	X .req \X
-	Y .req \Y
-	ext	v25.16b, X.16b, X.16b, #8
-	ext	v26.16b, Y.16b, Y.16b, #8
-	eor	v25.16b, v25.16b, X.16b
-	eor	v26.16b, v26.16b, Y.16b
-	pmull2	HI.1q, X.2d, Y.2d
-	pmull	LO.1q, X.1d, Y.1d
-	pmull	MI.1q, v25.1d, v26.1d
-	.unreq X
-	.unreq Y
-.endm
-
-/*
- * Computes the 256-bit polynomial represented by LO, HI, MI. Stores
- * the result in PL, PH.
- * [PH : PL] =
- *   [HI_1 : HI_1 + HI_0 + MI_1 + LO_1 : HI_0 + MI_0 + LO_1 + LO_0 : LO_0]
- */
-.macro karatsuba2
-	// v4 = [HI_1 + MI_1 : HI_0 + MI_0]
-	eor	v4.16b, HI.16b, MI.16b
-	// v4 = [HI_1 + MI_1 + LO_1 : HI_0 + MI_0 + LO_0]
-	eor	v4.16b, v4.16b, LO.16b
-	// v5 = [HI_0 : LO_1]
-	ext	v5.16b, LO.16b, HI.16b, #8
-	// v4 = [HI_1 + HI_0 + MI_1 + LO_1 : HI_0 + MI_0 + LO_1 + LO_0]
-	eor	v4.16b, v4.16b, v5.16b
-	// HI = [HI_0 : HI_1]
-	ext	HI.16b, HI.16b, HI.16b, #8
-	// LO = [LO_0 : LO_1]
-	ext	LO.16b, LO.16b, LO.16b, #8
-	// PH = [HI_1 : HI_1 + HI_0 + MI_1 + LO_1]
-	ext	PH.16b, v4.16b, HI.16b, #8
-	// PL = [HI_0 + MI_0 + LO_1 + LO_0 : LO_0]
-	ext	PL.16b, LO.16b, v4.16b, #8
-.endm
-
-/*
- * Computes the 128-bit reduction of PH : PL. Stores the result in dest.
- *
- * This macro computes p(x) mod g(x) where p(x) is in montgomery form and g(x) =
- * x^128 + x^127 + x^126 + x^121 + 1.
- *
- * We have a 256-bit polynomial PH : PL = P_3 : P_2 : P_1 : P_0 that is the
- * product of two 128-bit polynomials in Montgomery form.  We need to reduce it
- * mod g(x).  Also, since polynomials in Montgomery form have an "extra" factor
- * of x^128, this product has two extra factors of x^128.  To get it back into
- * Montgomery form, we need to remove one of these factors by dividing by x^128.
- *
- * To accomplish both of these goals, we add multiples of g(x) that cancel out
- * the low 128 bits P_1 : P_0, leaving just the high 128 bits. Since the low
- * bits are zero, the polynomial division by x^128 can be done by right
- * shifting.
- *
- * Since the only nonzero term in the low 64 bits of g(x) is the constant term,
- * the multiple of g(x) needed to cancel out P_0 is P_0 * g(x).  The CPU can
- * only do 64x64 bit multiplications, so split P_0 * g(x) into x^128 * P_0 +
- * x^64 * g*(x) * P_0 + P_0, where g*(x) is bits 64-127 of g(x).  Adding this to
- * the original polynomial gives P_3 : P_2 + P_0 + T_1 : P_1 + T_0 : 0, where T
- * = T_1 : T_0 = g*(x) * P_0.  Thus, bits 0-63 got "folded" into bits 64-191.
- *
- * Repeating this same process on the next 64 bits "folds" bits 64-127 into bits
- * 128-255, giving the answer in bits 128-255. This time, we need to cancel P_1
- * + T_0 in bits 64-127. The multiple of g(x) required is (P_1 + T_0) * g(x) *
- * x^64. Adding this to our previous computation gives P_3 + P_1 + T_0 + V_1 :
- * P_2 + P_0 + T_1 + V_0 : 0 : 0, where V = V_1 : V_0 = g*(x) * (P_1 + T_0).
- *
- * So our final computation is:
- *   T = T_1 : T_0 = g*(x) * P_0
- *   V = V_1 : V_0 = g*(x) * (P_1 + T_0)
- *   p(x) / x^{128} mod g(x) = P_3 + P_1 + T_0 + V_1 : P_2 + P_0 + T_1 + V_0
- *
- * The implementation below saves a XOR instruction by computing P_1 + T_0 : P_0
- * + T_1 and XORing into dest, rather than separately XORing P_1 : P_0 and T_0 :
- * T_1 into dest.  This allows us to reuse P_1 + T_0 when computing V.
- */
-.macro montgomery_reduction dest
-	DEST .req \dest
-	// TMP_V = T_1 : T_0 = P_0 * g*(x)
-	pmull	TMP_V.1q, PL.1d, GSTAR.1d
-	// TMP_V = T_0 : T_1
-	ext	TMP_V.16b, TMP_V.16b, TMP_V.16b, #8
-	// TMP_V = P_1 + T_0 : P_0 + T_1
-	eor	TMP_V.16b, PL.16b, TMP_V.16b
-	// PH = P_3 + P_1 + T_0 : P_2 + P_0 + T_1
-	eor	PH.16b, PH.16b, TMP_V.16b
-	// TMP_V = V_1 : V_0 = (P_1 + T_0) * g*(x)
-	pmull2	TMP_V.1q, TMP_V.2d, GSTAR.2d
-	eor	DEST.16b, PH.16b, TMP_V.16b
-	.unreq DEST
-.endm
-
-/*
- * Compute Polyval on 8 blocks.
- *
- * If reduce is set, also computes the montgomery reduction of the
- * previous full_stride call and XORs with the first message block.
- * (m_0 + REDUCE(PL, PH))h^8 + ... + m_7h^1.
- * I.e., the first multiplication uses m_0 + REDUCE(PL, PH) instead of m_0.
- *
- * Sets PL, PH.
- */
-.macro full_stride reduce
-	eor		LO.16b, LO.16b, LO.16b
-	eor		MI.16b, MI.16b, MI.16b
-	eor		HI.16b, HI.16b, HI.16b
-
-	ld1		{M0.16b, M1.16b, M2.16b, M3.16b}, [MSG], #64
-	ld1		{M4.16b, M5.16b, M6.16b, M7.16b}, [MSG], #64
-
-	karatsuba1 M7 KEY1
-	.if \reduce
-	pmull	TMP_V.1q, PL.1d, GSTAR.1d
-	.endif
-
-	karatsuba1 M6 KEY2
-	.if \reduce
-	ext	TMP_V.16b, TMP_V.16b, TMP_V.16b, #8
-	.endif
-
-	karatsuba1 M5 KEY3
-	.if \reduce
-	eor	TMP_V.16b, PL.16b, TMP_V.16b
-	.endif
-
-	karatsuba1 M4 KEY4
-	.if \reduce
-	eor	PH.16b, PH.16b, TMP_V.16b
-	.endif
-
-	karatsuba1 M3 KEY5
-	.if \reduce
-	pmull2	TMP_V.1q, TMP_V.2d, GSTAR.2d
-	.endif
-
-	karatsuba1 M2 KEY6
-	.if \reduce
-	eor	SUM.16b, PH.16b, TMP_V.16b
-	.endif
-
-	karatsuba1 M1 KEY7
-	eor	M0.16b, M0.16b, SUM.16b
-
-	karatsuba1 M0 KEY8
-	karatsuba2
-.endm
-
-/*
- * Handle any extra blocks after full_stride loop.
- */
-.macro partial_stride
-	add	KEY_POWERS, KEY_START, #(STRIDE_BLOCKS << 4)
-	sub	KEY_POWERS, KEY_POWERS, BLOCKS_LEFT, lsl #4
-	ld1	{KEY1.16b}, [KEY_POWERS], #16
-
-	ld1	{TMP_V.16b}, [MSG], #16
-	eor	SUM.16b, SUM.16b, TMP_V.16b
-	karatsuba1_store KEY1 SUM
-	sub	BLOCKS_LEFT, BLOCKS_LEFT, #1
-
-	tst	BLOCKS_LEFT, #4
-	beq	.Lpartial4BlocksDone
-	ld1	{M0.16b, M1.16b,  M2.16b, M3.16b}, [MSG], #64
-	ld1	{KEY8.16b, KEY7.16b, KEY6.16b,	KEY5.16b}, [KEY_POWERS], #64
-	karatsuba1 M0 KEY8
-	karatsuba1 M1 KEY7
-	karatsuba1 M2 KEY6
-	karatsuba1 M3 KEY5
-.Lpartial4BlocksDone:
-	tst	BLOCKS_LEFT, #2
-	beq	.Lpartial2BlocksDone
-	ld1	{M0.16b, M1.16b}, [MSG], #32
-	ld1	{KEY8.16b, KEY7.16b}, [KEY_POWERS], #32
-	karatsuba1 M0 KEY8
-	karatsuba1 M1 KEY7
-.Lpartial2BlocksDone:
-	tst	BLOCKS_LEFT, #1
-	beq	.LpartialDone
-	ld1	{M0.16b}, [MSG], #16
-	ld1	{KEY8.16b}, [KEY_POWERS], #16
-	karatsuba1 M0 KEY8
-.LpartialDone:
-	karatsuba2
-	montgomery_reduction SUM
-.endm
-
-/*
- * Perform montgomery multiplication in GF(2^128) and store result in op1.
- *
- * Computes op1*op2*x^{-128} mod x^128 + x^127 + x^126 + x^121 + 1
- * If op1, op2 are in montgomery form, this computes the montgomery
- * form of op1*op2.
- *
- * void pmull_polyval_mul(u8 *op1, const u8 *op2);
- */
-SYM_FUNC_START(pmull_polyval_mul)
-	adr	TMP, .Lgstar
-	ld1	{GSTAR.2d}, [TMP]
-	ld1	{v0.16b}, [x0]
-	ld1	{v1.16b}, [x1]
-	karatsuba1_store v0 v1
-	karatsuba2
-	montgomery_reduction SUM
-	st1	{SUM.16b}, [x0]
-	ret
-SYM_FUNC_END(pmull_polyval_mul)
-
-/*
- * Perform polynomial evaluation as specified by POLYVAL.  This computes:
- *	h^n * accumulator + h^n * m_0 + ... + h^1 * m_{n-1}
- * where n=nblocks, h is the hash key, and m_i are the message blocks.
- *
- * x0 - pointer to precomputed key powers h^8 ... h^1
- * x1 - pointer to message blocks
- * x2 - number of blocks to hash
- * x3 - pointer to accumulator
- *
- * void pmull_polyval_update(const struct polyval_ctx *ctx, const u8 *in,
- *			     size_t nblocks, u8 *accumulator);
- */
-SYM_FUNC_START(pmull_polyval_update)
-	adr	TMP, .Lgstar
-	mov	KEY_START, KEY_POWERS
-	ld1	{GSTAR.2d}, [TMP]
-	ld1	{SUM.16b}, [ACCUMULATOR]
-	subs	BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
-	blt .LstrideLoopExit
-	ld1	{KEY8.16b, KEY7.16b, KEY6.16b, KEY5.16b}, [KEY_POWERS], #64
-	ld1	{KEY4.16b, KEY3.16b, KEY2.16b, KEY1.16b}, [KEY_POWERS], #64
-	full_stride 0
-	subs	BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
-	blt .LstrideLoopExitReduce
-.LstrideLoop:
-	full_stride 1
-	subs	BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
-	bge	.LstrideLoop
-.LstrideLoopExitReduce:
-	montgomery_reduction SUM
-.LstrideLoopExit:
-	adds	BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
-	beq	.LskipPartial
-	partial_stride
-.LskipPartial:
-	st1	{SUM.16b}, [ACCUMULATOR]
-	ret
-SYM_FUNC_END(pmull_polyval_update)