X-Git-Url: https://git.distorted.org.uk/~mdw/catacomb/blobdiff_plain/43ea7558955ca453d4687c62d12efcb585124e8a..8f2287ef5c05d496fcb9b012629af007fe56f897:/symm/rijndael-x86ish-aesni.S

diff --git a/symm/rijndael-x86ish-aesni.S b/symm/rijndael-x86ish-aesni.S
index 8090bca6..6d9b3b22 100644
--- a/symm/rijndael-x86ish-aesni.S
+++ b/symm/rijndael-x86ish-aesni.S
@@ -25,20 +25,21 @@
 /// MA 02111-1307, USA.
 
 ///--------------------------------------------------------------------------
-/// External definitions.
+/// Preliminaries.
 
 #include "config.h"
 #include "asm-common.h"
 
-	.globl	F(abort)
-	.globl	F(rijndael_rcon)
+	.arch	.aes
 
-///--------------------------------------------------------------------------
-/// Main code.
+	.extern	F(abort)
+	.extern	F(rijndael_rcon)
 
-	.arch	.aes
 	.text
 
+///--------------------------------------------------------------------------
+/// Main code.
+
 /// The AESNI instructions implement a little-endian version of AES, but
 /// Catacomb's internal interface presents as big-endian so as to work better
 /// with things like GCM.  We therefore maintain the round keys in
@@ -61,6 +62,12 @@
 ///--------------------------------------------------------------------------
 /// Key setup.
 
+FUNC(rijndael_setup_x86ish_aesni_avx)
+	vzeroupper		      // avoid penalty on `legacy' XMM access
+  endprologue
+	// and drop through...
+ENDFUNC
+
 FUNC(rijndael_setup_x86ish_aesni)
 
 #define SI WHOLE(si)
@@ -85,10 +92,10 @@ FUNC(rijndael_setup_x86ish_aesni)
 #  define BLKOFF edx			// block size in bytes
 
 	// Stack the caller's registers.
-	push	ebp
-	push	ebx
-	push	esi
-	push	edi
+	pushreg	ebp
+	pushreg	ebx
+	pushreg	esi
+	pushreg	edi
 
 	// Set up our own variables.
 	mov	CTX, [esp + 20]		// context base pointer
@@ -138,17 +145,16 @@ FUNC(rijndael_setup_x86ish_aesni)
 
 	// We'll need the index registers, which belong to the caller in this
 	// ABI.
-	push	rsi
-	  .seh_pushreg rsi
-	push	rdi
-	  .seh_pushreg rdi
-  .seh_endprologue
+	pushreg	rsi
+	pushreg	rdi
 
 	// Move arguments to more useful places.
 	mov	rsi, r8			// key material
 	mov	CTX, rcx		// context base pointer
 #endif
 
+  endprologue
+
 	// The initial round key material is taken directly from the input
 	// key, so copy it over.
 #if CPUFAM_AMD64 && ABI_SYSV
@@ -206,16 +212,16 @@ FUNC(rijndael_setup_x86ish_aesni)
 	// Fourth word of the cycle, and seven or eight words of key.  Do a
 	// byte substitution.
 	movd	xmm0, eax
-	pshufd	xmm0, xmm0, SHUF(2, 1, 0, 3)
+	pshufd	xmm0, xmm0, SHUF(3, 0, 1, 2)
 	aeskeygenassist xmm1, xmm0, 0
 	movd	eax, xmm1
 	jmp	2f
 
 	// First word of the cycle.  This is the complicated piece.
 1:	movd	xmm0, eax
-	pshufd	xmm0, xmm0, SHUF(0, 3, 2, 1)
+	pshufd	xmm0, xmm0, SHUF(1, 2, 3, 0)
 	aeskeygenassist xmm1, xmm0, 0
-	pshufd	xmm1, xmm1, SHUF(2, 1, 0, 3)
+	pshufd	xmm1, xmm1, SHUF(3, 0, 1, 2)
 	movd	eax, xmm1
 	xor	al, [RCON]
 	inc	RCON
@@ -321,21 +327,24 @@ FUNC(rijndael_setup_x86ish_aesni)
 
 9:	// All done.
 #if CPUFAM_X86
-	pop	edi
-	pop	esi
-	pop	ebx
-	pop	ebp
+	popreg	edi
+	popreg	esi
+	popreg	ebx
+	popreg	ebp
 #endif
 #if CPUFAM_AMD64 && ABI_WIN
-	pop	rdi
-	pop	rsi
+	popreg	rdi
+	popreg	rsi
 #endif
 	ret
 
-	.align	16
-endswap_block:
+ENDFUNC
+
+INTFUNC(endswap_block)
 	// End-swap NKW words starting at SI.  The end-swapping table is
 	// already loaded into XMM5; and it's OK to work in 16-byte chunks.
+  endprologue
+
 	mov	ecx, NKW
 0:	movdqu	xmm1, [SI]
 	pshufb	xmm1, xmm5
@@ -343,8 +352,11 @@ endswap_block:
 	add	SI, 16
 	sub	ecx, 4
 	ja	0b
+
 	ret
 
+ENDFUNC
+
 #undef CTX
 #undef BLKSZ
 #undef SI
@@ -356,12 +368,16 @@ endswap_block:
 #undef LRK
 #undef BLKOFF
 
-ENDFUNC
-
 ///--------------------------------------------------------------------------
 /// Encrypting and decrypting blocks.
 
 .macro	encdec	op, aes, koff
+  FUNC(rijndael_\op\()_x86ish_aesni_avx)
+	vzeroupper			// avoid XMM penalties
+  endprologue
+	// and drop through...
+  ENDFUNC
+
   FUNC(rijndael_\op\()_x86ish_aesni)
 
 #if CPUFAM_X86
@@ -393,9 +409,10 @@ ENDFUNC
 #  define SRC rdx
 #  define DST r8
 #  define NR eax
-  .seh_endprologue
 #endif
 
+  endprologue
+
 	// Find the magic endianness-swapping table.
 	ldgot	ecx
 	movdqa	xmm5, [INTADDR(endswap_tab, ecx)]
@@ -509,17 +526,21 @@ ENDFUNC
 ///--------------------------------------------------------------------------
 /// Random utilities.
 
-	.align	16
+INTFUNC(bogus)
 	// Abort the process because of a programming error.  Indirecting
 	// through this point serves several purposes: (a) by CALLing, rather
 	// than branching to, `abort', we can save the return address, which
 	// might at least provide a hint as to what went wrong; (b) we don't
 	// have conditional CALLs (and they'd be big anyway); and (c) we can
 	// write a HLT here as a backstop against `abort' being mad.
-bogus:	callext	F(abort)
+  endprologue
+
+	callext	F(abort)
 0:	hlt
 	jmp	0b
 
+ENDFUNC
+
 ///--------------------------------------------------------------------------
 /// Data tables.