[catacomb] / symm / salsa20-arm-neon.S

/// -*- mode: asm; asm-comment-char: ?/ -*-
///
/// Fancy SIMD implementation of Salsa20 for ARM
///
/// (c) 2016 Straylight/Edgeware
///

///----- Licensing notice ---------------------------------------------------
///
/// This file is part of Catacomb.
///
/// Catacomb is free software; you can redistribute it and/or modify
/// it under the terms of the GNU Library General Public License as
/// published by the Free Software Foundation; either version 2 of the
/// License, or (at your option) any later version.
///
/// Catacomb is distributed in the hope that it will be useful,
/// but WITHOUT ANY WARRANTY; without even the implied warranty of
/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
/// GNU Library General Public License for more details.
///
/// You should have received a copy of the GNU Library General Public
/// License along with Catacomb; if not, write to the Free
/// Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
/// MA 02111-1307, USA.

///--------------------------------------------------------------------------
/// Preliminaries.

#include "config.h"
#include "asm-common.h"

	.arch	armv7-a
	.fpu	neon

	.text

///--------------------------------------------------------------------------
/// Main code.

FUNC(salsa20_core_arm_neon)

	// Arguments are in registers.
	// r0 is the number of rounds to perform
	// r1 points to the input matrix
	// r2 points to the output matrix

	// First job is to slurp the matrix into the SIMD registers.  The
	// words have already been permuted conveniently to make them line up
	// better for SIMD processing.
	//
	// The textbook arrangement of the matrix is this.
	//
	//	[C K K K]
	//	[K C N N]
	//	[T T C K]
	//	[K K K C]
	//
	// But we've rotated the columns up so that the main diagonal with
	// the constants on it end up in the first row, giving something more
	// like
	//
	//	[C C C C]
	//	[K T K K]
	//	[T K K N]
	//	[K K N K]
	//
	// so the transformation looks like this:
	//
	//	[ 0  1  2  3]		[ 0  5 10 15] (a, q8)
	//	[ 4  5  6  7]    -->	[ 4  9 14  3] (b, q9)
	//	[ 8  9 10 11]		[ 8 13  2  7] (c, q10)
	//	[12 13 14 15]		[12  1  6 11] (d, q11)
	//
	// We need a copy for later.  Rather than waste time copying them by
	// hand, we'll use the three-address nature of the instruction set.
	// But this means that the main loop is offset by a bit.
	vldmia	r1, {QQ(q12, q15)}

	// Apply a column quarterround to each of the columns simultaneously,
	// moving the results to their working registers.  Alas, there
	// doesn't seem to be a packed word rotate, so we have to synthesize
	// it.

	// b ^= (a + d) <<<  7
	vadd.u32 q0, q12, q15
	vshl.u32 q1, q0, #7
	vshr.u32 q0, q0, #25
	vorr	q0, q0, q1
	veor	q9, q13, q0

	// c ^= (b + a) <<<  9
	vadd.u32 q0, q9, q12
	vshl.u32 q1, q0, #9
	vshr.u32 q0, q0, #23
	vorr	q0, q0, q1
	veor	q10, q14, q0

	// d ^= (c + b) <<< 13
	vadd.u32 q0, q10, q9
	 vext.32 q9, q9, q9, #3
	vshl.u32 q1, q0, #13
	vshr.u32 q0, q0, #19
	vorr	q0, q0, q1
	veor	q11, q15, q0

	// a ^= (d + c) <<< 18
	vadd.u32 q0, q11, q10
	 vext.32 q10, q10, q10, #2
	 vext.32 q11, q11, q11, #1
	vshl.u32 q1, q0, #18
	vshr.u32 q0, q0, #14
	vorr	q0, q0, q1
	veor	q8, q12, q0

0:
	// The transpose conveniently only involves reordering elements of
	// individual rows, which can be done quite easily, and reordering
	// the rows themselves, which is a trivial renaming.  It doesn't
	// involve any movement of elements between rows.
	//
	//	[ 0  5 10 15]		[ 0  5 10 15] (a, q8)
	//	[ 4  9 14  3]	 -->	[ 1  6 11 12] (b, q11)
	//	[ 8 13	2  7]		[ 2  7	8 13] (c, q10)
	//	[12  1	6 11]		[ 3  4	9 14] (d, q9)
	//
	// The reorderings have been pushed upwards to reduce delays.

	// Apply the row quarterround to each of the columns (yes!)
	// simultaneously.

	// b ^= (a + d) <<<  7
	vadd.u32 q0, q8, q9
	vshl.u32 q1, q0, #7
	vshr.u32 q0, q0, #25
	vorr	q0, q0, q1
	veor	q11, q11, q0

	// c ^= (b + a) <<<  9
	vadd.u32 q0, q11, q8
	vshl.u32 q1, q0, #9
	vshr.u32 q0, q0, #23
	vorr	q0, q0, q1
	veor	q10, q10, q0

	// d ^= (c + b) <<< 13
	vadd.u32 q0, q10, q11
	 vext.32 q11, q11, q11, #3
	vshl.u32 q1, q0, #13
	vshr.u32 q0, q0, #19
	vorr	q0, q0, q1
	veor	q9, q9, q0

	// a ^= (d + c) <<< 18
	vadd.u32 q0, q9, q10
	 vext.32 q10, q10, q10, #2
	 vext.32 q9, q9, q9, #1
	vshl.u32 q1, q0, #18
	vshr.u32 q0, q0, #14
	vorr	q0, q0, q1
	veor	q8, q8, q0

	// We had to undo the transpose ready for the next loop.  Again, push
	// back the reorderings to reduce latency.  Decrement the loop
	// counter and see if we should go round again.
	subs	r0, r0, #2
	bls	9f

	// Do the first half of the next round because this loop is offset.

	// b ^= (a + d) <<<  7
	vadd.u32 q0, q8, q11
	vshl.u32 q1, q0, #7
	vshr.u32 q0, q0, #25
	vorr	q0, q0, q1
	veor	q9, q9, q0

	// c ^= (b + a) <<<  9
	vadd.u32 q0, q9, q8
	vshl.u32 q1, q0, #9
	vshr.u32 q0, q0, #23
	vorr	q0, q0, q1
	veor	q10, q10, q0

	// d ^= (c + b) <<< 13
	vadd.u32 q0, q10, q9
	 vext.32 q9, q9, q9, #3
	vshl.u32 q1, q0, #13
	vshr.u32 q0, q0, #19
	vorr	q0, q0, q1
	veor	q11, q11, q0

	// a ^= (d + c) <<< 18
	vadd.u32 q0, q11, q10
	 vext.32 q10, q10, q10, #2
	 vext.32 q11, q11, q11, #1
	vshl.u32 q1, q0, #18
	vshr.u32 q0, q0, #14
	vorr	q0, q0, q1
	veor	q8, q8, q0

	b	0b

	// Almost there.  Firstly the feedfoward addition.  Also, establish a
	// constant which will be useful later.
9:	vadd.u32 q0, q8, q12			//  0,  5, 10, 15
	 vmov.i64 q12, #0xffffffff		// = (0, -1; 0, -1)
	vadd.u32 q1, q9, q13			//  4,  9, 14,  3
	vadd.u32 q2, q10, q14			//  8, 13,  2,  7
	vadd.u32 q3, q11, q15			// 12,  1,  6, 11

	// Next we must undo the permutation which was already applied to the
	// input.  The core trick is from Dan Bernstein's `armneon3'
	// implementation, but with a lot of liposuction.
	vmov	q15, q0

	// Sort out the columns by pairs.
	vbif	q0, q3, q12			//  0,  1, 10, 11
	vbif	q3, q2, q12			// 12, 13,  6,  7
	vbif	q2, q1, q12			//  8,  9,  2,  3
	vbif	q1, q15, q12			//  4,  5, 14, 15

	// Now fix up the remaining discrepancies.
	vswp	D1(q0), D1(q2)
	vswp	D1(q1), D1(q3)

	// And with that, we're done.
	vstmia	r2, {QQ(q0, q3)}
	bx	r14

ENDFUNC

///----- That's all, folks --------------------------------------------------
Commit	Line	Data
704d59c8 MW	1	/// -- mode: asm; asm-comment-char: ?/ --
	2	///
	3	/// Fancy SIMD implementation of Salsa20 for ARM
	4	///
	5	/// (c) 2016 Straylight/Edgeware
	6	///
	7
	8	///----- Licensing notice ---------------------------------------------------
	9	///
	10	/// This file is part of Catacomb.
	11	///
	12	/// Catacomb is free software; you can redistribute it and/or modify
	13	/// it under the terms of the GNU Library General Public License as
	14	/// published by the Free Software Foundation; either version 2 of the
	15	/// License, or (at your option) any later version.
	16	///
	17	/// Catacomb is distributed in the hope that it will be useful,
	18	/// but WITHOUT ANY WARRANTY; without even the implied warranty of
	19	/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	20	/// GNU Library General Public License for more details.
	21	///
	22	/// You should have received a copy of the GNU Library General Public
	23	/// License along with Catacomb; if not, write to the Free
	24	/// Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
	25	/// MA 02111-1307, USA.
	26
	27	///--------------------------------------------------------------------------
df07f2c0	28	/// Preliminaries.
704d59c8 MW	29
	30	#include "config.h"
	31	#include "asm-common.h"
	32
704d59c8 MW	33	.arch armv7-a
704d59c8 MW	34	.fpu neon
df07f2c0	35
bc9ac7eb	36	.text
704d59c8	37
df07f2c0	38	///--------------------------------------------------------------------------
6d7e6032	39	/// Main code.
df07f2c0	40
704d59c8 MW	41	FUNC(salsa20_core_arm_neon)
	42
	43	// Arguments are in registers.
	44	// r0 is the number of rounds to perform
	45	// r1 points to the input matrix
	46	// r2 points to the output matrix
	47
	48	// First job is to slurp the matrix into the SIMD registers. The
	49	// words have already been permuted conveniently to make them line up
	50	// better for SIMD processing.
	51	//
	52	// The textbook arrangement of the matrix is this.
	53	//
	54	// [C K K K]
	55	// [K C N N]
	56	// [T T C K]
	57	// [K K K C]
	58	//
	59	// But we've rotated the columns up so that the main diagonal with
	60	// the constants on it end up in the first row, giving something more
	61	// like
	62	//
	63	// [C C C C]
	64	// [K T K K]
	65	// [T K K N]
	66	// [K K N K]
	67	//
	68	// so the transformation looks like this:
	69	//
	70	// [ 0 1 2 3] [ 0 5 10 15] (a, q8)
	71	// [ 4 5 6 7] --> [ 4 9 14 3] (b, q9)
	72	// [ 8 9 10 11] [ 8 13 2 7] (c, q10)
	73	// [12 13 14 15] [12 1 6 11] (d, q11)
	74	//
704d59c8 MW	75	// We need a copy for later. Rather than waste time copying them by
	76	// hand, we'll use the three-address nature of the instruction set.
	77	// But this means that the main loop is offset by a bit.
43ea7558	78	vldmia r1, {QQ(q12, q15)}
704d59c8 MW	79
	80	// Apply a column quarterround to each of the columns simultaneously,
	81	// moving the results to their working registers. Alas, there
	82	// doesn't seem to be a packed word rotate, so we have to synthesize
	83	// it.
	84
	85	// b ^= (a + d) <<< 7
	86	vadd.u32 q0, q12, q15
	87	vshl.u32 q1, q0, #7
	88	vshr.u32 q0, q0, #25
	89	vorr q0, q0, q1
	90	veor q9, q13, q0
	91
	92	// c ^= (b + a) <<< 9
	93	vadd.u32 q0, q9, q12
	94	vshl.u32 q1, q0, #9
	95	vshr.u32 q0, q0, #23
	96	vorr q0, q0, q1
	97	veor q10, q14, q0
	98
	99	// d ^= (c + b) <<< 13
	100	vadd.u32 q0, q10, q9
ff1e7aff	101	vext.32 q9, q9, q9, #3
704d59c8 MW	102	vshl.u32 q1, q0, #13
	103	vshr.u32 q0, q0, #19
	104	vorr q0, q0, q1
	105	veor q11, q15, q0
	106
	107	// a ^= (d + c) <<< 18
	108	vadd.u32 q0, q11, q10
ff1e7aff MW	109	vext.32 q10, q10, q10, #2
ff1e7aff MW	110	vext.32 q11, q11, q11, #1
704d59c8 MW	111	vshl.u32 q1, q0, #18
	112	vshr.u32 q0, q0, #14
	113	vorr q0, q0, q1
	114	veor q8, q12, q0
	115
	116	0:
	117	// The transpose conveniently only involves reordering elements of
	118	// individual rows, which can be done quite easily, and reordering
	119	// the rows themselves, which is a trivial renaming. It doesn't
	120	// involve any movement of elements between rows.
	121	//
	122	// [ 0 5 10 15] [ 0 5 10 15] (a, q8)
	123	// [ 4 9 14 3] --> [ 1 6 11 12] (b, q11)
	124	// [ 8 13 2 7] [ 2 7 8 13] (c, q10)
	125	// [12 1 6 11] [ 3 4 9 14] (d, q9)
	126	//
	127	// The reorderings have been pushed upwards to reduce delays.
	128
	129	// Apply the row quarterround to each of the columns (yes!)
	130	// simultaneously.
	131
	132	// b ^= (a + d) <<< 7
	133	vadd.u32 q0, q8, q9
	134	vshl.u32 q1, q0, #7
	135	vshr.u32 q0, q0, #25
	136	vorr q0, q0, q1
	137	veor q11, q11, q0
	138
	139	// c ^= (b + a) <<< 9
	140	vadd.u32 q0, q11, q8
	141	vshl.u32 q1, q0, #9
	142	vshr.u32 q0, q0, #23
	143	vorr q0, q0, q1
	144	veor q10, q10, q0
	145
	146	// d ^= (c + b) <<< 13
	147	vadd.u32 q0, q10, q11
70bc6059	148	vext.32 q11, q11, q11, #3
704d59c8 MW	149	vshl.u32 q1, q0, #13
	150	vshr.u32 q0, q0, #19
	151	vorr q0, q0, q1
	152	veor q9, q9, q0
	153
	154	// a ^= (d + c) <<< 18
	155	vadd.u32 q0, q9, q10
70bc6059 MW	156	vext.32 q10, q10, q10, #2
70bc6059 MW	157	vext.32 q9, q9, q9, #1
704d59c8 MW	158	vshl.u32 q1, q0, #18
	159	vshr.u32 q0, q0, #14
	160	vorr q0, q0, q1
	161	veor q8, q8, q0
	162
	163	// We had to undo the transpose ready for the next loop. Again, push
	164	// back the reorderings to reduce latency. Decrement the loop
	165	// counter and see if we should go round again.
	166	subs r0, r0, #2
	167	bls 9f
	168
	169	// Do the first half of the next round because this loop is offset.
	170
	171	// b ^= (a + d) <<< 7
	172	vadd.u32 q0, q8, q11
	173	vshl.u32 q1, q0, #7
	174	vshr.u32 q0, q0, #25
	175	vorr q0, q0, q1
	176	veor q9, q9, q0
	177
	178	// c ^= (b + a) <<< 9
	179	vadd.u32 q0, q9, q8
	180	vshl.u32 q1, q0, #9
	181	vshr.u32 q0, q0, #23
	182	vorr q0, q0, q1
	183	veor q10, q10, q0
	184
	185	// d ^= (c + b) <<< 13
	186	vadd.u32 q0, q10, q9
70bc6059	187	vext.32 q9, q9, q9, #3
704d59c8 MW	188	vshl.u32 q1, q0, #13
	189	vshr.u32 q0, q0, #19
	190	vorr q0, q0, q1
	191	veor q11, q11, q0
	192
	193	// a ^= (d + c) <<< 18
	194	vadd.u32 q0, q11, q10
70bc6059 MW	195	vext.32 q10, q10, q10, #2
70bc6059 MW	196	vext.32 q11, q11, q11, #1
704d59c8 MW	197	vshl.u32 q1, q0, #18
	198	vshr.u32 q0, q0, #14
	199	vorr q0, q0, q1
	200	veor q8, q8, q0
	201
	202	b 0b
	203
f48fb6a6 MW	204	// Almost there. Firstly the feedfoward addition. Also, establish a
f48fb6a6 MW	205	// constant which will be useful later.
3cb47d27	206	9: vadd.u32 q0, q8, q12 // 0, 5, 10, 15
f2cd5445	207	vmov.i64 q12, #0xffffffff // = (0, -1; 0, -1)
f48fb6a6 MW	208	vadd.u32 q1, q9, q13 // 4, 9, 14, 3
	209	vadd.u32 q2, q10, q14 // 8, 13, 2, 7
	210	vadd.u32 q3, q11, q15 // 12, 1, 6, 11
3cb47d27 MW	211
3cb47d27 MW	212	// Next we must undo the permutation which was already applied to the
f48fb6a6 MW	213	// input. The core trick is from Dan Bernstein's `armneon3'
	214	// implementation, but with a lot of liposuction.
	215	vmov q15, q0
	216
	217	// Sort out the columns by pairs.
	218	vbif q0, q3, q12 // 0, 1, 10, 11
	219	vbif q3, q2, q12 // 12, 13, 6, 7
	220	vbif q2, q1, q12 // 8, 9, 2, 3
	221	vbif q1, q15, q12 // 4, 5, 14, 15
	222
	223	// Now fix up the remaining discrepancies.
	224	vswp D1(q0), D1(q2)
	225	vswp D1(q1), D1(q3)
704d59c8 MW	226
704d59c8 MW	227	// And with that, we're done.
43ea7558	228	vstmia r2, {QQ(q0, q3)}
704d59c8 MW	229	bx r14
	230
	231	ENDFUNC
	232
	233	///----- That's all, folks --------------------------------------------------