[catacomb] / symm / rijndael-arm-crypto.S

/// -*- mode: asm; asm-comment-char: ?/ -*-
///
/// ARM crypto-extension-based implementation of Rijndael
///
/// (c) 2016 Straylight/Edgeware
///

///----- Licensing notice ---------------------------------------------------
///
/// This file is part of Catacomb.
///
/// Catacomb is free software; you can redistribute it and/or modify
/// it under the terms of the GNU Library General Public License as
/// published by the Free Software Foundation; either version 2 of the
/// License, or (at your option) any later version.
///
/// Catacomb is distributed in the hope that it will be useful,
/// but WITHOUT ANY WARRANTY; without even the implied warranty of
/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
/// GNU Library General Public License for more details.
///
/// You should have received a copy of the GNU Library General Public
/// License along with Catacomb; if not, write to the Free
/// Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
/// MA 02111-1307, USA.

///--------------------------------------------------------------------------
/// External definitions.

#include "config.h"
#include "asm-common.h"

	.extern	F(abort)
	.extern	F(rijndael_rcon)

///--------------------------------------------------------------------------
/// Main code.

	.arch	armv8-a
	.fpu	crypto-neon-fp-armv8

/// The ARM crypto extension implements a little-endian version of AES
/// (though the manual doesn't actually spell this out and you have to
/// experiment), but Catacomb's internal interface presents as big-endian so
/// as to work better with things like GCM.  We therefore maintain the round
/// keys in little-endian form, and have to end-swap blocks in and out.
///
/// For added amusement, the crypto extension doesn't implement the larger-
/// block versions of Rijndael, so we have to end-swap the keys if we're
/// preparing for one of those.

	// Useful constants.
	.equ	maxrounds, 16		// maximum number of rounds
	.equ	maxblksz, 32		// maximum block size, in bytes
	.equ	kbufsz, maxblksz*(maxrounds + 1) // size of key-sched buffer

	// Context structure.
	.equ	nr, 0			// number of rounds
	.equ	w, nr + 4		// encryption key words
	.equ	wi, w + kbufsz		// decryption key words

///--------------------------------------------------------------------------
/// Key setup.

FUNC(rijndael_setup_arm_crypto)

	// Arguments:
	//	r0 = pointer to context
	//	r1 = block size in words
	//	r2 = pointer to key material
	//	r3 = key size in words

	pushreg	r4-r9, r14

	// The initial round key material is taken directly from the input
	// key, so copy it over.  Unfortunately, the key material is not
	// guaranteed to be aligned in any especially useful way, so we must
	// sort this out.
	add	r9, r0, #w
	mov	r14, r3
	ands	r6, r2, #3
	beq	1f
	mov	r6, r6, lsl #3
	rsb	r7, r6, #32
	bic	r2, r2, #3
	ldr	r4, [r2], #4

0:	ldr	r5, [r2], #4
	mov	r4, r4, lsr r6
	orr	r4, r5, lsl r7
	str	r4, [r9], #4
	subs	r14, r14, #1
	movhi	r4, r5
	bhi	0b
	b	9f

1:	ldr	r4, [r2], #4
	str	r4, [r9], #4
	subs	r14, r14, #1
	bhi	1b

	// Find out other useful things and prepare for the main loop.
9:	ldr	r7, [r0, #nr]		// number of rounds
	mla	r2, r1, r7, r1		// total key size in words
	leaextq	r5, rijndael_rcon	// round constants
	sub	r8, r2, r3		// minus what we've copied already
	veor	q1, q1			// all-zero register for the key
	add	r8, r9, r8, lsl #2	// limit of the key buffer
	mov	r12, #0			// position in current cycle

	// Main key expansion loop.  Dispatch according to the position in
	// the cycle.
0:	ldr	r6, [r9, -r3, lsl #2]	// word from previous cycle
	cmp	r12, #0			// first word of the cycle?
	beq	1f
	cmp	r12, #4			// fourth word of the cycle?
	bne	2f
	cmp	r3, #7			// seven or eight words of key?
	bcc	2f

	// Fourth word of the cycle, seven or eight words of key.  We must do
	// the byte substitution.
	vdup.32	q0, r4
	aese.8	q0, q1			// effectively, just SubBytes
	vmov.32	r4, d0[0]
	b	2f

	// First word of the cycle.  Byte substitution, rotation, and round
	// constant.
1:	ldrb	r14, [r5], #1		// next round constant
	ldr	r6, [r9, -r3, lsl #2]
	vdup.32	q0, r4
	aese.8	q0, q1			// effectively, just SubBytes
	vmov.32	r4, d0[0]
	eor	r4, r14, r4, ror #8

	// Common ending: mix in the word from the previous cycle and store.
2:	eor	r4, r4, r6
	str	r4, [r9], #4

	// Prepare for the next iteration.  If we're done, then stop; if
	// we've finished a cycle then reset the counter.
	add	r12, r12, #1
	cmp	r9, r8
	bcs	9f
	cmp	r12, r3
	movcs	r12, #0
	b	0b

	// Next job is to construct the decryption keys.  The keys for the
	// first and last rounds don't need to be mangled, but the remaining
	// ones do -- and they all need to be reordered too.
	//
	// The plan of action, then, is to copy the final encryption round's
	// keys into place first, then to do each of the intermediate rounds
	// in reverse order, and finally do the first round.
	//
	// Do all the heavy lifting with NEON registers.  The order we're
	// doing this in means that it's OK if we read or write too much, and
	// there's easily enough buffer space for the over-enthusiastic reads
	// and writes because the context has space for 32-byte blocks, which
	// is our maximum and an exact fit for two Q-class registers.
9:	add	r5, r0, #wi
	add	r4, r0, #w
	add	r4, r4, r2, lsl #2
	sub	r4, r4, r1, lsl #2		// last round's keys

	// Copy the last encryption round's keys.
	teq	r1, #4
	vldmiaeq r4, {d0, d1}
	vldmiane r4, {d0-d3}
	vstmiaeq r5, {d0, d1}
	vstmiane r5, {d0-d3}

	// Update the loop variables and stop if we've finished.
0:	sub	r4, r4, r1, lsl #2
	add	r5, r5, r1, lsl #2
	subs	r7, r7, #1
	beq	9f

	// Do another middle round's keys...
	teq	r1, #4
	vldmiaeq r4, {d0, d1}
	vldmiane r4, {d0-d3}
	aesimc.8 q0, q0
	vstmiaeq r5, {d0, d1}
	beq	0b
	aesimc.8 q1, q1
	vstmia	r5, {d0-d3}
	b	0b

	// Finally do the first encryption round.
9:	teq	r1, #4
	vldmiaeq r4, {d0, d1}
	vldmiane r4, {d0-d3}
	vstmiaeq r5, {d0, d1}
	vstmiane r5, {d0-d3}

	// If the block size is not exactly four words then we must end-swap
	// everything.  We can use fancy NEON toys for this.
	beq	9f

	// End-swap the encryption keys.
	add	r1, r0, #w
	bl	endswap_block

	// And the decryption keys
	add	r1, r0, #wi
	bl	endswap_block

	// All done.
9:	popreg	r4-r9, pc

ENDFUNC

INTFUNC(endswap_block)
	// End-swap R2 words starting at R1.  R1 is clobbered; R2 is not.
	// It's OK to work in 16-byte chunks.

	mov	r4, r2
0:	vldmia	r1, {d0, d1}
	vrev32.8 q0, q0
	vstmia	r1!, {d0, d1}
	subs	r4, r4, #4
	bhi	0b
	bx	r14

ENDFUNC

///--------------------------------------------------------------------------
/// Encrypting and decrypting blocks.

.macro	encdec	op, aes, mc, koff
  FUNC(rijndael_\op\()_arm_crypto)

	// Arguments:
	//	r0 = pointer to context
	//	r1 = pointer to input block
	//	r2 = pointer to output block

	// Set things up ready.
	ldr	r3, [r0, #nr]
	add	r0, r0, #\koff
	vldmia	r1, {d0, d1}
	vrev32.8 q0, q0

	// Check the number of rounds and dispatch.
	sub	r3, r3, #10
	cmp	r3, #5
	addlo	pc, pc, r3, lsl #2
	callext	F(abort)

	b	10f
	b	11f
	b	12f
	b	13f
	b	14f

	// Eleven rounds.
11:	vldmia	r0!, {d16, d17}
	\aes\().8 q0, q8
	\mc\().8 q0, q0
	b	10f

	// Twelve rounds.
12:	vldmia	r0!, {d16-d19}
	\aes\().8 q0, q8
	\mc\().8 q0, q0
	\aes\().8 q0, q9
	\mc\().8 q0, q0
	b	10f

	// Thirteen rounds.
13:	vldmia	r0!, {d16-d21}
	\aes\().8 q0, q8
	\mc\().8 q0, q0
	\aes\().8 q0, q9
	\mc\().8 q0, q0
	\aes\().8 q0, q10
	\mc\().8 q0, q0
	b	10f

	// Fourteen rounds.  (Drops through to the ten round case because
	// this is the next most common.)
14:	vldmia	r0!, {d16-d23}
	\aes\().8 q0, q8
	\mc\().8 q0, q0
	\aes\().8 q0, q9
	\mc\().8 q0, q0
	\aes\().8 q0, q10
	\mc\().8 q0, q0
	\aes\().8 q0, q11
	\mc\().8 q0, q0
	// Drop through...

	// Ten rounds.
10:	vldmia	r0!, {d16-d25}
	\aes\().8 q0, q8
	\mc\().8 q0, q0
	\aes\().8 q0, q9
	\mc\().8 q0, q0
	\aes\().8 q0, q10
	\mc\().8 q0, q0
	\aes\().8 q0, q11
	\mc\().8 q0, q0
	\aes\().8 q0, q12
	\mc\().8 q0, q0

	vldmia	r0!, {d16-d27}
	\aes\().8 q0, q8
	\mc\().8 q0, q0
	\aes\().8 q0, q9
	\mc\().8 q0, q0
	\aes\().8 q0, q10
	\mc\().8 q0, q0
	\aes\().8 q0, q11
	\mc\().8 q0, q0

	// Final round has no MixColumns, but is followed by final whitening.
	\aes\().8 q0, q12
	veor	q0, q0, q13

	// All done.
	vrev32.8 q0, q0
	vstmia	r2, {d0, d1}
	bx	r14

  ENDFUNC
.endm

	encdec	eblk, aese, aesmc, w
	encdec	dblk, aesd, aesimc, wi

///----- That's all, folks --------------------------------------------------
Commit	Line	Data
26e182fc MW	1	/// -- mode: asm; asm-comment-char: ?/ --
	2	///
	3	/// ARM crypto-extension-based implementation of Rijndael
	4	///
	5	/// (c) 2016 Straylight/Edgeware
	6	///
	7
	8	///----- Licensing notice ---------------------------------------------------
	9	///
	10	/// This file is part of Catacomb.
	11	///
	12	/// Catacomb is free software; you can redistribute it and/or modify
	13	/// it under the terms of the GNU Library General Public License as
	14	/// published by the Free Software Foundation; either version 2 of the
	15	/// License, or (at your option) any later version.
	16	///
	17	/// Catacomb is distributed in the hope that it will be useful,
	18	/// but WITHOUT ANY WARRANTY; without even the implied warranty of
	19	/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	20	/// GNU Library General Public License for more details.
	21	///
	22	/// You should have received a copy of the GNU Library General Public
	23	/// License along with Catacomb; if not, write to the Free
	24	/// Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
	25	/// MA 02111-1307, USA.
	26
	27	///--------------------------------------------------------------------------
	28	/// External definitions.
	29
	30	#include "config.h"
	31	#include "asm-common.h"
	32
1a517bb3 MW	33	.extern F(abort)
1a517bb3 MW	34	.extern F(rijndael_rcon)
26e182fc MW	35
	36	///--------------------------------------------------------------------------
	37	/// Main code.
	38
	39	.arch armv8-a
	40	.fpu crypto-neon-fp-armv8
	41
	42	/// The ARM crypto extension implements a little-endian version of AES
	43	/// (though the manual doesn't actually spell this out and you have to
	44	/// experiment), but Catacomb's internal interface presents as big-endian so
	45	/// as to work better with things like GCM. We therefore maintain the round
	46	/// keys in little-endian form, and have to end-swap blocks in and out.
	47	///
	48	/// For added amusement, the crypto extension doesn't implement the larger-
	49	/// block versions of Rijndael, so we have to end-swap the keys if we're
	50	/// preparing for one of those.
	51
	52	// Useful constants.
	53	.equ maxrounds, 16 // maximum number of rounds
	54	.equ maxblksz, 32 // maximum block size, in bytes
43ea7558	55	.equ kbufsz, maxblksz*(maxrounds + 1) // size of key-sched buffer
26e182fc MW	56
	57	// Context structure.
	58	.equ nr, 0 // number of rounds
	59	.equ w, nr + 4 // encryption key words
	60	.equ wi, w + kbufsz // decryption key words
	61
	62	///--------------------------------------------------------------------------
	63	/// Key setup.
	64
	65	FUNC(rijndael_setup_arm_crypto)
	66
	67	// Arguments:
	68	// r0 = pointer to context
	69	// r1 = block size in words
	70	// r2 = pointer to key material
	71	// r3 = key size in words
	72
ae4fc08a	73	pushreg r4-r9, r14
26e182fc MW	74
	75	// The initial round key material is taken directly from the input
	76	// key, so copy it over. Unfortunately, the key material is not
	77	// guaranteed to be aligned in any especially useful way, so we must
	78	// sort this out.
	79	add r9, r0, #w
	80	mov r14, r3
c82543b7	81	ands r6, r2, #3
26e182fc	82	beq 1f
c82543b7 MW	83	mov r6, r6, lsl #3
c82543b7 MW	84	rsb r7, r6, #32
26e182fc	85	bic r2, r2, #3
c82543b7	86	ldr r4, [r2], #4
26e182fc	87
c82543b7 MW	88	0: ldr r5, [r2], #4
	89	mov r4, r4, lsr r6
	90	orr r4, r5, lsl r7
	91	str r4, [r9], #4
26e182fc	92	subs r14, r14, #1
c82543b7	93	movhi r4, r5
26e182fc MW	94	bhi 0b
	95	b 9f
	96
c82543b7 MW	97	1: ldr r4, [r2], #4
c82543b7 MW	98	str r4, [r9], #4
26e182fc MW	99	subs r14, r14, #1
	100	bhi 1b
	101
	102	// Find out other useful things and prepare for the main loop.
aec6bc36	103	9: ldr r7, [r0, #nr] // number of rounds
26e182fc	104	mla r2, r1, r7, r1 // total key size in words
26e182fc MW	105	leaextq r5, rijndael_rcon // round constants
	106	sub r8, r2, r3 // minus what we've copied already
	107	veor q1, q1 // all-zero register for the key
	108	add r8, r9, r8, lsl #2 // limit of the key buffer
16021451	109	mov r12, #0 // position in current cycle
26e182fc	110
16021451 MW	111	// Main key expansion loop. Dispatch according to the position in
	112	// the cycle.
	113	0: ldr r6, [r9, -r3, lsl #2] // word from previous cycle
	114	cmp r12, #0 // first word of the cycle?
	115	beq 1f
	116	cmp r12, #4 // fourth word of the cycle?
	117	bne 2f
	118	cmp r3, #7 // seven or eight words of key?
	119	bcc 2f
	120
	121	// Fourth word of the cycle, seven or eight words of key. We must do
	122	// the byte substitution.
26e182fc MW	123	vdup.32 q0, r4
	124	aese.8 q0, q1 // effectively, just SubBytes
	125	vmov.32 r4, d0[0]
16021451	126	b 2f
26e182fc	127
16021451 MW	128	// First word of the cycle. Byte substitution, rotation, and round
	129	// constant.
	130	1: ldrb r14, [r5], #1 // next round constant
26e182fc	131	ldr r6, [r9, -r3, lsl #2]
26e182fc MW	132	vdup.32 q0, r4
	133	aese.8 q0, q1 // effectively, just SubBytes
	134	vmov.32 r4, d0[0]
16021451	135	eor r4, r14, r4, ror #8
26e182fc	136
16021451 MW	137	// Common ending: mix in the word from the previous cycle and store.
16021451 MW	138	2: eor r4, r4, r6
26e182fc	139	str r4, [r9], #4
26e182fc	140
16021451 MW	141	// Prepare for the next iteration. If we're done, then stop; if
	142	// we've finished a cycle then reset the counter.
	143	add r12, r12, #1
26e182fc	144	cmp r9, r8
f71eed58	145	bcs 9f
16021451 MW	146	cmp r12, r3
16021451 MW	147	movcs r12, #0
f71eed58	148	b 0b
26e182fc MW	149
	150	// Next job is to construct the decryption keys. The keys for the
	151	// first and last rounds don't need to be mangled, but the remaining
	152	// ones do -- and they all need to be reordered too.
	153	//
	154	// The plan of action, then, is to copy the final encryption round's
	155	// keys into place first, then to do each of the intermediate rounds
	156	// in reverse order, and finally do the first round.
	157	//
	158	// Do all the heavy lifting with NEON registers. The order we're
	159	// doing this in means that it's OK if we read or write too much, and
	160	// there's easily enough buffer space for the over-enthusiastic reads
	161	// and writes because the context has space for 32-byte blocks, which
	162	// is our maximum and an exact fit for two Q-class registers.
f71eed58	163	9: add r5, r0, #wi
26e182fc MW	164	add r4, r0, #w
	165	add r4, r4, r2, lsl #2
	166	sub r4, r4, r1, lsl #2 // last round's keys
	167
	168	// Copy the last encryption round's keys.
	169	teq r1, #4
	170	vldmiaeq r4, {d0, d1}
	171	vldmiane r4, {d0-d3}
	172	vstmiaeq r5, {d0, d1}
	173	vstmiane r5, {d0-d3}
	174
	175	// Update the loop variables and stop if we've finished.
f71eed58	176	0: sub r4, r4, r1, lsl #2
26e182fc MW	177	add r5, r5, r1, lsl #2
26e182fc MW	178	subs r7, r7, #1
f71eed58	179	beq 9f
26e182fc MW	180
	181	// Do another middle round's keys...
	182	teq r1, #4
	183	vldmiaeq r4, {d0, d1}
	184	vldmiane r4, {d0-d3}
	185	aesimc.8 q0, q0
	186	vstmiaeq r5, {d0, d1}
f71eed58	187	beq 0b
26e182fc MW	188	aesimc.8 q1, q1
26e182fc MW	189	vstmia r5, {d0-d3}
f71eed58	190	b 0b
26e182fc MW	191
26e182fc MW	192	// Finally do the first encryption round.
f71eed58	193	9: teq r1, #4
26e182fc MW	194	vldmiaeq r4, {d0, d1}
	195	vldmiane r4, {d0-d3}
	196	vstmiaeq r5, {d0, d1}
	197	vstmiane r5, {d0-d3}
	198
	199	// If the block size is not exactly four words then we must end-swap
	200	// everything. We can use fancy NEON toys for this.
f71eed58	201	beq 9f
26e182fc MW	202
	203	// End-swap the encryption keys.
	204	add r1, r0, #w
	205	bl endswap_block
	206
	207	// And the decryption keys
	208	add r1, r0, #wi
	209	bl endswap_block
	210
	211	// All done.
ae4fc08a	212	9: popreg r4-r9, pc
26e182fc	213
1a517bb3 MW	214	ENDFUNC
	215
	216	INTFUNC(endswap_block)
26e182fc MW	217	// End-swap R2 words starting at R1. R1 is clobbered; R2 is not.
26e182fc MW	218	// It's OK to work in 16-byte chunks.
1a517bb3	219
26e182fc MW	220	mov r4, r2
	221	0: vldmia r1, {d0, d1}
	222	vrev32.8 q0, q0
	223	vstmia r1!, {d0, d1}
	224	subs r4, r4, #4
	225	bhi 0b
	226	bx r14
	227
	228	ENDFUNC
	229
	230	///--------------------------------------------------------------------------
	231	/// Encrypting and decrypting blocks.
	232
6717fd12 MW	233	.macro encdec op, aes, mc, koff
6717fd12 MW	234	FUNC(rijndael_\op\()_arm_crypto)
26e182fc MW	235
	236	// Arguments:
	237	// r0 = pointer to context
	238	// r1 = pointer to input block
	239	// r2 = pointer to output block
	240
	241	// Set things up ready.
	242	ldr r3, [r0, #nr]
6717fd12	243	add r0, r0, #\koff
26e182fc MW	244	vldmia r1, {d0, d1}
	245	vrev32.8 q0, q0
	246
6717fd12 MW	247	// Check the number of rounds and dispatch.
	248	sub r3, r3, #10
	249	cmp r3, #5
	250	addlo pc, pc, r3, lsl #2
26e182fc MW	251	callext F(abort)
26e182fc MW	252
6717fd12 MW	253	b 10f
	254	b 11f
	255	b 12f
	256	b 13f
	257	b 14f
	258
	259	// Eleven rounds.
	260	11: vldmia r0!, {d16, d17}
	261	\aes\().8 q0, q8
	262	\mc\().8 q0, q0
	263	b 10f
	264
	265	// Twelve rounds.
	266	12: vldmia r0!, {d16-d19}
	267	\aes\().8 q0, q8
	268	\mc\().8 q0, q0
	269	\aes\().8 q0, q9
	270	\mc\().8 q0, q0
	271	b 10f
	272
	273	// Thirteen rounds.
	274	13: vldmia r0!, {d16-d21}
	275	\aes\().8 q0, q8
	276	\mc\().8 q0, q0
	277	\aes\().8 q0, q9
	278	\mc\().8 q0, q0
	279	\aes\().8 q0, q10
	280	\mc\().8 q0, q0
	281	b 10f
	282
	283	// Fourteen rounds. (Drops through to the ten round case because
	284	// this is the next most common.)
	285	14: vldmia r0!, {d16-d23}
	286	\aes\().8 q0, q8
	287	\mc\().8 q0, q0
	288	\aes\().8 q0, q9
	289	\mc\().8 q0, q0
	290	\aes\().8 q0, q10
	291	\mc\().8 q0, q0
	292	\aes\().8 q0, q11
	293	\mc\().8 q0, q0
	294	// Drop through...
	295
	296	// Ten rounds.
	297	10: vldmia r0!, {d16-d25}
	298	\aes\().8 q0, q8
	299	\mc\().8 q0, q0
	300	\aes\().8 q0, q9
	301	\mc\().8 q0, q0
	302	\aes\().8 q0, q10
	303	\mc\().8 q0, q0
	304	\aes\().8 q0, q11
	305	\mc\().8 q0, q0
	306	\aes\().8 q0, q12
	307	\mc\().8 q0, q0
	308
	309	vldmia r0!, {d16-d27}
	310	\aes\().8 q0, q8
	311	\mc\().8 q0, q0
	312	\aes\().8 q0, q9
	313	\mc\().8 q0, q0
	314	\aes\().8 q0, q10
	315	\mc\().8 q0, q0
	316	\aes\().8 q0, q11
317	\mc\().8 q0, q0
318
319	// Final round has no MixColumns, but is followed by final whitening.
320	\aes\().8 q0, q12
321	veor q0, q0, q13
26e182fc MW	322
	323	// All done.
	324	vrev32.8 q0, q0
	325	vstmia r2, {d0, d1}
	326	bx r14
	327
6717fd12 MW	328	ENDFUNC
6717fd12 MW	329	.endm
26e182fc	330
6717fd12 MW	331	encdec eblk, aese, aesmc, w
6717fd12 MW	332	encdec dblk, aesd, aesimc, wi
26e182fc MW	333
26e182fc MW	334	///----- That's all, folks --------------------------------------------------