progs/perftest.c: Use from Glibc syscall numbers.

[catacomb] / symm / gcm-arm-crypto.S

/// -*- mode: asm; asm-comment-char: ?/ -*-
///
/// GCM acceleration for ARM processors
///
/// (c) 2019 Straylight/Edgeware
///

///----- Licensing notice ---------------------------------------------------
///
/// This file is part of Catacomb.
///
/// Catacomb is free software: you can redistribute it and/or modify it
/// under the terms of the GNU Library General Public License as published
/// by the Free Software Foundation; either version 2 of the License, or
/// (at your option) any later version.
///
/// Catacomb is distributed in the hope that it will be useful, but
/// WITHOUT ANY WARRANTY; without even the implied warranty of
/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
/// Library General Public License for more details.
///
/// You should have received a copy of the GNU Library General Public
/// License along with Catacomb.  If not, write to the Free Software
/// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
/// USA.

///--------------------------------------------------------------------------
/// Preliminaries.

#include "config.h"
#include "asm-common.h"

        .arch   armv8-a
        .fpu    crypto-neon-fp-armv8

        .text

///--------------------------------------------------------------------------
/// Multiplication macros.

        // The good news is that we have a fancy instruction to do the
        // multiplications.  The bad news is that it's not particularly well-
        // suited to the job.
        //
        // For one thing, it only does a 64-bit multiplication, so in general
        // we'll need to synthesize the full-width multiply by hand.  For
        // another thing, it doesn't help with the reduction, so we have to
        // do that by hand too.  And, finally, GCM has crazy bit ordering,
        // and the instruction does nothing useful for that at all.
        //
        // Focusing on that last problem first: the bits aren't in monotonic
        // significance order unless we permute them.  If we reverse the byte
        // order, then we'll have the bits in monotonic order, but backwards,
        // so the degree-0 coefficient will be in the most-significant bit.
        //
        // This is less of a difficulty than it seems at first, because
        // algebra.  Suppose we are given u = SUM_{0<=i<n} u_i t^i and v =
        // SUM_{0<=j<n} v_j t^j; then
        //
        //      u v = SUM_{0<=i,j<n} u_i v_j t^{i+j}
        //
        // Suppose instead that we're given ũ = SUM_{0<=i<n} u_{n-i-1} t^i
        // and ṽ = SUM_{0<=j<n} v_{n-j-1} t^j, so the bits are backwards.
        // Then
        //
        //      ũ ṽ = SUM_{0<=i,j<n} u_{n-i-1} v_{n-j-1} t^{i+j}
        //          = SUM_{0<=i,j<n} u_i v_j t^{2n-2-(i+j)}
        //
        // which is almost the bit-reversal of u v, only it's shifted right
        // by one place.  Putting this another way, what we have is actually
        // the bit reversal of the product u v t.  We could get the correct
        // answer (modulo p(t)) if we'd sneakily divided one of the operands
        // by t before we started.  Conveniently, v is actually the secret
        // value k set up by the GCM `mktable' function, so we can arrange to
        // actually store k/t (mod p(t)) and then the product will come out
        // correct (modulo p(t)) and we won't have anything more to worry
        // about here.
        //
        // That was important to think about, but there's not a great deal to
        // do about it yet other than to convert what we've got from the
        // blockcipher's byte-ordering convention to our big-endian
        // convention.  Since this depends on the blockcipher convention,
        // we'll leave the caller to cope with this: the macros here will
        // assume that the operands are in `register' format, which is the
        // same as the external representation, except that the bytes within
        // each 64-bit piece are reversed.  In the commentary, pieces of
        // polynomial are numbered according to the degree of the
        // coefficients, so the unit coefficient of some polynomial a is in
        // a_0.
        //
        // The commentary for `mul128' is the most detailed.  The other
        // macros assume that you've already read and understood that.

.macro  mul128
        // Enter with u and v in q0 and q1 respectively; leave with z = u v
        // in q0.  Clobbers q1--q3, q8, q9.

        // First for the double-precision multiplication.  It's tempting to
        // use Karatsuba's identity here, but I suspect that loses more in
        // the shifting, bit-twiddling, and dependency chains that it gains
        // in saving a multiplication which otherwise pipelines well.
        // q0 =                         // (u_1; u_0)
        // q1 =                         // (v_1; v_0)
        vmull.p64 q2, d1, d2            // u_1 v_0
        vmull.p64 q3, d0, d3            // u_0 v_1
        vmull.p64 q8, d1, d3            // (t_1; x_3) = u_1 v_1
        vmull.p64 q9, d0, d2            // (x_0; t_0) = u_0 v_0

        // Arrange the pieces to form a double-precision polynomial.
        veor    q2, q2, q3              // (m_0; m_1) = u_0 v_1 + u_1 v_0
        veor    d17, d17, d4            // x_2 = t_1 + m_1
        veor    d18, d18, d5            // x_1 = t_0 + m_0
        // q8 =                         // (x_2; x_3)
        // q9 =                         // (x_0; x_1)

        // One-and-a-half problems remain.
        //
        // The full-size problem is that the result needs to be reduced
        // modulo p(t) = t^128 + t^7 + t^2 + t + 1.  Let R = t^128 = t^7 +
        // t^2 + t + 1 in our field.  So far, we've calculated z_0 and z_1
        // such that z_0 + z_1 R = u v using the identity R = t^128: now we
        // must collapse the two halves of y together using the other
        // identity R = t^7 + t^2 + t + 1.
        //
        // We do this by working on x_2 and x_3 separately, so consider x_i
        // for i = 2 or 3.  Certainly, x_i t^{64i} = x_i R t^{64(i-2) =
        // (t^7 + t^2 + t + 1) x_i t^{64(i-2)}, but we can't use that
        // directly without breaking up the 64-bit word structure.  Instead,
        // we start by considering just x_i t^7 t^{64(i-2)}, which again
        // looks tricky.  Now, split x_i = a_i + t^57 b_i, with deg a_i < 57;
        // then
        //
        //      x_i t^7 t^{64(i-2)} = a_i t^7 t^{64(i-2)} + b_i t^{64(i-1)}
        //
        // We can similarly decompose x_i t^2 and x_i t into a pair of 64-bit
        // contributions to the t^{64(i-2)} and t^{64(i-1)} words, but the
        // splits are different.  This is lovely, with one small snag: when
        // we do this to x_3, we end up with a contribution back into the
        // t^128 coefficient word.  But notice that only the low seven bits
        // of this word are affected, so there's no knock-on contribution
        // into the t^64 word.  Therefore, if we handle the high bits of each
        // word together, and then the low bits, everything will be fine.

        // First, shift the high bits down.
        vshl.u64 q2, q8, #63            // the b_i for t
        vshl.u64 q3, q8, #62            // the b_i for t^2
        vshl.u64 q0, q8, #57            // the b_i for t^7
        veor    q2, q2, q3              // add them all together
        veor    q2, q2, q0
        veor    d18, d18, d5            // contribution into low half
        veor    d17, d17, d4            // and high half

        // And then shift the low bits up.
        vshr.u64 q2, q8, #1
        vshr.u64 q3, q8, #2
        vshr.u64 q1, q8, #7
        veor    q8, q8, q9              // mix in the unit contribution
        veor    q2, q2, q3              // t and t^2 contribs
        veor    q1, q1, q8              // low, unit, and t^7 contribs
        veor    d1, d2, d4              // mix them together and swap halves
        veor    d0, d3, d5
.endm

.macro  mul64
        // Enter with u and v in the low halves of d0 and d1 respectively;
        // leave with z = u v in d0.  Clobbers d1--d5.

        // The multiplication is thankfully easy.
        vmull.p64 q0, d0, d1            // u v

        // Now we must reduce.  This is essentially the same as the 128-bit
        // case above, but mostly simpler because everything is smaller.  The
        // polynomial this time is p(t) = t^64 + t^4 + t^3 + t + 1.

        // First, shift the high bits down.
        vshl.u64 d2, d0, #63            // b_i for t
        vshl.u64 d3, d0, #61            // b_i for t^3
        vshl.u64 d4, d0, #60            // b_i for t^4
        veor    d2, d2, d3              // add them all together
        veor    d2, d2, d4
        veor    d0, d0, d2              // contribution back into high half

        // And then shift the low bits up.
        vshr.u64 d2, d0, #1
        vshr.u64 d3, d0, #3
        vshr.u64 d4, d0, #4
        veor    d0, d0, d1              // mix in the unit contribution
        veor    d2, d2, d3              // t and t^3 contribs
        veor    d0, d0, d4              // low, unit, and t^4
        veor    d0, d0, d2              // mix them together and we're done
.endm

.macro  mul96
        // Enter with u and v in the most-significant three words of q0 and
        // q1 respectively, and zero in the low words, and zero in q15; leave
        // with z = u v in the high three words of q0, and /junk/ in the low
        // word.  Clobbers q1--q3, q8, q9.

        // This is an inconvenient size.  There's nothing for it but to do
        // four multiplications, as if for the 128-bit case.
        // q0 =                         // (u_2; u_0 + u_1 t^32)
        // q1 =                         // (v_2; v_0 + v_1 t^32)
        vmull.p64 q8, d1, d2            // u_2 (v_0 + v_1 t^32) = e_0
        vmull.p64 q9, d0, d3            // v_2 (u_0 + u_1 t^32) = e_1
        vmull.p64 q3, d1, d3            // u_2 v_2 t^64 = d = (d; 0)
        vmull.p64 q0, d0, d2            // u_0 v_0 + (u_0 v_1 + u_1 v_0) t^32
                                        //   + u_1 v_1 t^64 = f

        // Extract the high and low halves of the 192-bit result.  The answer
        // we want is d t^128 + e t^64 + f, where e = e_0 + e_1.  The low 96
        // bits of the answer will end up in q0, and the high 96 bits will
        // end up in q1; we'll need both of these to have zero in their
        // bottom 32 bits.
        //
        // Here, bot(x) is the low 96 bits of a 192-bit quantity x, arranged
        // in the low 96 bits of a SIMD register, with junk in the top 32
        // bits; and top(x) is the high 96 bits, also arranged in the low 96
        // bits of a register, with /zero/ in the top 32 bits.
        veor    q8, q8, q9              // e_0 + e_1 = e
        vshr128 q1, q3, 32              // top(d t^128)
        vext.8  d19, d16, d17, #4       // top(e t^64)
        vshl.u64 d16, d0, #32           // top(f), sort of
        veor    d3, d3, d19             // q1 = top(d t^128 + e t^64)
        veor    d0, d0, d17             // q0 = bot([d t^128] + e t^64 + f)
        veor    d3, d3, d16             // q1 = top(d t^128 + e t^64 + f)

        // Finally, the reduction.  This is essentially the same as the
        // 128-bit case, except that the polynomial is p(t) = t^96 + t^10 +
        // t^9 + t^6 + 1.  The degrees are larger but not enough to cause
        // trouble for the general approach.

        // First, shift the high bits down.
        vshl.u32 q2, q1, #26            // b_i for t^6
        vshl.u32 q3, q1, #23            // b_i for t^9
        vshl.u32 q8, q1, #22            // b_i for t^10
        veor    q2, q2, q3              // add them all together
        veor    q2, q2, q8
        vshl128 q3, q2, 64              // contribution into high half
        vshr128 q2, q2, 32              // and low half
        veor    q1, q1, q3              // mix them in
        veor    q0, q0, q2

        // And then shift the low bits up.
        vshr.u32 q2, q1, #6
        vshr.u32 q3, q1, #9
        veor    q0, q0, q1              // mix in the unit contribution
        vshr.u32 q8, q1, #10
        veor    q2, q2, q3              // mix together t^6 and t^9
        veor    q0, q0, q8              // mix in t^10
        veor    q0, q0, q2              // and the rest

        // And finally swap the two halves.
        vswp    d0, d1
.endm

.macro  mul192
        // Enter with u and v in d0--d2 and d3--d5 respectively; leave
        // with z = u v in d0--d2.  Clobbers q8--q15.

        // Start multiplying and accumulating pieces of product.
        // (d0; d1; d2) =               // (u_0; u_1; u_2)
        // (d3; d4; d5) =               // (v_0; v_1; v_2)
        vmull.p64 q10, d0, d3           // e = u_0 v_0

        vmull.p64 q12, d0, d4           //     u_0 v_1
        vmull.p64 q13, d1, d3           //     u_1 v_0

        vmull.p64 q9, d0, d5            //     u_0 v_2
        vmull.p64 q14, d1, d4           //     u_1 v_1
        vmull.p64 q15, d2, d3           //     u_2 v_0
         veor   q12, q12, q13           // d = u_0 v_1 + u_1 v_0

        vmull.p64 q11, d1, d5           //     u_1 v_2
        vmull.p64 q13, d2, d4           //     u_2 v_1
         veor   q9, q9, q14             //     u_0 v_2 + u_1 v_1

        vmull.p64 q8, d2, d5            // a = u_2 v_2
         veor   q9, q9, q15             // c = u_0 v_2 + u_1 v_1 + u_2 v_0
         veor   q11, q11, q13           // b = u_1 v_2 + u_2 v_1

        // Piece the product together.
        veor    d17, d17, d22  //  q8 = // (x_4; x_5)
        veor    d18, d18, d23
        veor    d19, d19, d24  //  q9 = // (x_2; x_3)
        veor    d20, d20, d25  // q10 = // (x_0; x_1)

        // Next, the reduction.  Our polynomial this time is p(x) = t^192 +
        // t^7 + t^2 + t + 1.  Yes, the magic numbers are the same as the
        // 128-bit case.  I don't know why.

        // First, shift the high bits down.
        // q8 =                         // (y_4; y_5)
        // q9 =                         // (y_2; y_3)
        // q10 =                        // (y_0; y_1)
        vshl.u64 q11, q8, #63           // (y_4; y_5) b_i for t
        vshl.u64 d28, d18, #63          // y_3 b_i for t
        vshl.u64 q12, q8, #62           // (y_4; y_5) b_i for t^2
        vshl.u64 d29, d18, #62          // y_3 b_i for t^2
        vshl.u64 q13, q8, #57           // (y_4; y_5) b_i for t^7
        vshl.u64 d30, d18, #57          // y_3 b_i for t^7
        veor    q11, q11, q12           // mix them all together
        veor    d28, d28, d29
        veor    q11, q11, q13
        veor    d28, d28, d30
        veor    q9, q9, q11
        veor    d20, d20, d28

        // And finally shift the low bits up.  Also, switch the order of the
        // pieces for output.
        // q8 =                         // (y'_4; y'_5)
        // q9 =                         // (y'_2; y'_3)
        // q10 =                        // (y'_0; y'_1)
        vshr.u64 q11, q8, #1            // (y_4; y_5) a_i for t
        vshr.u64 d28, d18, #1           // y'_3 a_i for t
        vshr.u64 q12, q8, #2            // (y_4; y_5) a_i for t^2
        vshr.u64 d29, d18, #2           // y'_3 a_i for t^2
        vshr.u64 q13, q8, #7            // (y_4; y_5) a_i for t^7
        vshr.u64 d30, d18, #7           // y'_3 a_i for t^7
        veor    q8, q8, q11
        veor    d18, d18, d28
        veor    q12, q12, q13
        veor    d29, d29, d30
        veor    q8, q8, q12
        veor    d18, d18, d29
        veor    d0, d21, d18
        veor    d1, d20, d17
        veor    d2, d19, d16
.endm

.macro  mul256
        // Enter with u and v in q0/q1 and q2/q3 respectively; leave
        // with z = u v in q0/q1.  Clobbers q8--q15.

        // Now it's starting to look worthwhile to do Karatsuba.  Suppose
        // u = u_0 + u_1 B and v = v_0 + v_1 B.  Then
        //
        //      u v = (u_0 v_0) + (u_0 v_1 + u_1 v_0) B + (u_1 v_1) B^2
        //
        // Name these coefficients of B^i be a, b, and c, respectively, and
        // let r = u_0 + u_1 and s = v_0 + v_1.  Then observe that
        //
        //      q = r s = (u_0 + u_1) (v_0 + v_1)
        //        = (u_0 v_0) + (u1 v_1) + (u_0 v_1 + u_1 v_0)
        //        = a + c + b
        //
        // The first two terms we've already calculated; the last is the
        // remaining one we want.  We'll set B = t^128.  We know how to do
        // 128-bit multiplications already, and Karatsuba is too annoying
        // there, so there'll be 12 multiplications altogether, rather than
        // the 16 we'd have if we did this the naïve way.
        // q0 =                         // u_0 = (u_01; u_00)
        // q1 =                         // u_1 = (u_11; u_10)
        // q2 =                         // v_0 = (v_01; v_00)
        // q3 =                         // v_1 = (v_11; v_10)

        veor    q8, q0, q1              // u_* = (u_01 + u_11; u_00 + u_10)
        veor    q9, q2, q3              // v_* = (v_01 + v_11; v_00 + v_10)

        // Start by building the cross product, q = u_* v_*.
        vmull.p64 q14, d16, d19         // u_*0 v_*1
        vmull.p64 q15, d17, d18         // u_*1 v_*0
        vmull.p64 q12, d17, d19         // u_*1 v_*1
        vmull.p64 q13, d16, d18         // u_*0 v_*0
        veor    q14, q14, q15           // u_*0 v_*1 + u_*1 v_*0
        veor    d25, d25, d28  // q12 = // q_1
        veor    d26, d26, d29  // q13 = // q_0

        // Next, work on the low half, a = u_0 v_0.
        vmull.p64 q14, d0, d5           // u_00 v_01
        vmull.p64 q15, d1, d4           // u_01 v_00
        vmull.p64 q10, d1, d5           // u_01 v_01
        vmull.p64 q11, d0, d4           // u_00 v_00
        veor    q14, q14, q15           // u_00 v_01 + u_01 v_00
        veor    d21, d21, d28  // q10 = // a_1
        veor    d22, d22, d29  // q11 = // a_0

        // Mix the pieces we have so far.
        veor    q12, q12, q10
        veor    q13, q13, q11

        // Finally, the high half, c = u_1 v_1.
        vmull.p64 q14, d2, d7           // u_10 v_11
        vmull.p64 q15, d3, d6           // u_11 v_10
        vmull.p64 q8, d3, d7            // u_11 v_11
        vmull.p64 q9, d2, d6            // u_10 v_10
        veor    q14, q14, q15           // u_10 v_11 + u_11 v_10
        veor    d17, d17, d28  //  q8 = // c_1
        veor    d18, d18, d29  //  q9 = // c_0

        // Finish mixing the product together.
        veor    q12, q12, q8   // q12 = // b_1
        veor    q13, q13, q9   // q13 = // b_0
        veor    q9, q9, q12
        veor    q10, q10, q13

        // Now we must reduce.  This is essentially the same as the 192-bit
        // case above, but more complicated because everything is bigger.
        // The polynomial this time is p(t) = t^256 + t^10 + t^5 + t^2 + 1.

        // First, shift the high bits down.
        // q8 =                         // (y_6; y_7)
        // q9 =                         // (y_4; y_5)
        // q10 =                        // (y_2; y_3)
        // q11 =                        // (y_0; y_1)
        vshl.u64 q0, q8, #62            // (y_6; y_7) b_i for t^2
        vshl.u64 q12, q9, #62           // (y_4; y_5) b_i for t^2
        vshl.u64 q1, q8, #59            // (y_6; y_7) b_i for t^5
        vshl.u64 q13, q9, #59           // (y_4; y_5) b_i for t^5
        vshl.u64 q2, q8, #54            // (y_6; y_7) b_i for t^10
        vshl.u64 q14, q9, #54           // (y_4; y_5) b_i for t^10
        veor    q0, q0, q1              // mix the contributions together
        veor    q12, q12, q13
        veor    q0, q0, q2
        veor    q12, q12, q14
        veor    d19, d19, d0            // and combine into the lower pieces
        veor    d20, d20, d1
        veor    d21, d21, d24
        veor    d22, d22, d25

        // And then shift the low bits up.  Also, switch the order of the
        // pieces for output.
        // q8 =                         // (y'_6; y'_7)
        // q9 =                         // (y'_4; y'_5)
        // q10 =                        // (y'_2; y'_3)
        // q11 =                        // (y'_0; y'_1)
        vshr.u64 q0, q8, #2             // (y_6; y_7) a_i for t^2
        vshr.u64 q12, q9, #2            // (y'_4; y_5) a_i for t^2
        vshr.u64 q1, q8, #5             // (y_6; y_7) a_i for t^5
        vshr.u64 q13, q9, #5            // (y_4; y_5) a_i for t^5
        vshr.u64 q2, q8, #10            // (y_6; y_7) a_i for t^10
        vshr.u64 q14, q9, #10           // (y_4; y_5) a_i for t^10

        veor    q8, q8, q0              // mix the contributions together
        veor    q1, q1, q2
        veor    q9, q9, q12
        veor    q13, q13, q14
        veor    q8, q8, q1
        veor    q9, q9, q13
        veor    d3, d20, d16            // and output
        veor    d2, d21, d17
        veor    d1, d22, d18
        veor    d0, d23, d19
.endm

///--------------------------------------------------------------------------
/// Main code.

// There are a number of representations of field elements in this code and
// it can be confusing.
//
//   * The `external format' consists of a sequence of contiguous bytes in
//     memory called a `block'.  The GCM spec explains how to interpret this
//     block as an element of a finite field.  As discussed extensively, this
//     representation is very annoying for a number of reasons.  On the other
//     hand, this code never actually deals with it directly.
//
//   * The `register format' consists of one or more NEON registers,
//     depending on the block size.  The bytes in each 64-bit lane of these
//     registers are in reverse order, compared to the external format.
//
//   * The `words' format consists of a sequence of bytes, as in the
//     `external format', but, according to the blockcipher in use, the bytes
//     within each 32-bit word may be reversed (`big-endian') or not
//     (`little-endian').  Accordingly, there are separate entry points for
//     each variant, identified with `b' or `l'.

FUNC(gcm_mulk_128b_arm_crypto)
        // On entry, r0 points to a 128-bit field element A in big-endian
        // words format; r1 points to a field-element K in register format.
        // On exit, A is updated with the product A K.

        vld1.8  {q0}, [r0]
        vld1.8  {q1}, [r1]
        vrev64.32 q0, q0
        mul128
        vrev64.32 q0, q0
        vst1.8  {q0}, [r0]
        bx      r14
ENDFUNC

FUNC(gcm_mulk_128l_arm_crypto)
        // On entry, r0 points to a 128-bit field element A in little-endian
        // words format; r1 points to a field-element K in register format.
        // On exit, A is updated with the product A K.

        vld1.8  {q0}, [r0]
        vld1.8  {q1}, [r1]
        vrev64.8 q0, q0
        mul128
        vrev64.8 q0, q0
        vst1.8  {q0}, [r0]
        bx      r14
ENDFUNC

FUNC(gcm_mulk_64b_arm_crypto)
        // On entry, r0 points to a 64-bit field element A in big-endian
        // words format; r1 points to a field-element K in register format.
        // On exit, A is updated with the product A K.

        vld1.8  {d0}, [r0]
        vld1.8  {d1}, [r1]
        vrev64.32 d0, d0
        mul64
        vrev64.32 d0, d0
        vst1.8  {d0}, [r0]
        bx      r14
ENDFUNC

FUNC(gcm_mulk_64l_arm_crypto)
        // On entry, r0 points to a 64-bit field element A in little-endian
        // words format; r1 points to a field-element K in register format.
        // On exit, A is updated with the product A K.

        vld1.8  {d0}, [r0]
        vld1.8  {d1}, [r1]
        vrev64.8 d0, d0
        vzero
        mul64
        vrev64.8 d0, d0
        vst1.8  {d0}, [r0]
        bx      r14
ENDFUNC

FUNC(gcm_mulk_96b_arm_crypto)
        // On entry, r0 points to a 96-bit field element A in big-endian
        // words format; r1 points to a field-element K in register format.
        // On exit, A is updated with the product A K.

        ldr     r3, [r0, #8]
        mov     r12, #0
        vld1.8  {d0}, [r0]
        vld1.8  {q1}, [r1]
        vrev64.32 d0, d0
        vmov    d1, r12, r3
        vzero
        mul96
        vrev64.32 d0, d0
        vmov    r3, d1[1]
        vst1.8  {d0}, [r0]
        str     r3, [r0, #8]
        bx      r14
ENDFUNC

FUNC(gcm_mulk_96l_arm_crypto)
        // On entry, r0 points to a 128-bit field element A in little-endian
        // words format; r1 points to a field-element K in register format.
        // On exit, A is updated with the product A K.

        ldr     r3, [r0, #8]
        mov     r12, #0
        vld1.8  {d0}, [r0]
        vld1.8  {q1}, [r1]
        vmov    d1, r3, r12
        vrev64.8 q0, q0
        mul96
        vrev64.8 q0, q0
        vmov    r3, d1[0]
        vst1.8  {d0}, [r0]
        str     r3, [r0, #8]
        bx      r14
ENDFUNC

FUNC(gcm_mulk_192b_arm_crypto)
        // On entry, r0 points to a 192-bit field element A in big-endian
        // words format; r1 points to a field-element K in register format.
        // On exit, A is updated with the product A K.

        vld1.8  {d0-d2}, [r0]
        vld1.8  {d3-d5}, [r1]
        vrev64.32 q0, q0
        vrev64.32 d2, d2
        mul192
        vrev64.32 q0, q0
        vrev64.32 d2, d2
        vst1.8  {d0-d2}, [r0]
        bx      r14
ENDFUNC

FUNC(gcm_mulk_192l_arm_crypto)
        // On entry, r0 points to a 192-bit field element A in little-endian
        // words format; r1 points to a field-element K in register format.
        // On exit, A is updated with the product A K.

        vld1.8  {d0-d2}, [r0]
        vld1.8  {d3-d5}, [r1]
        vrev64.8 q0, q0
        vrev64.8 d2, d2
        mul192
        vrev64.8 q0, q0
        vrev64.8 d2, d2
        vst1.8  {d0-d2}, [r0]
        bx      r14
ENDFUNC

FUNC(gcm_mulk_256b_arm_crypto)
        // On entry, r0 points to a 256-bit field element A in big-endian
        // words format; r1 points to a field-element K in register format.
        // On exit, A is updated with the product A K.

        vld1.8  {q0, q1}, [r0]
        vld1.8  {q2, q3}, [r1]
        vrev64.32 q0, q0
        vrev64.32 q1, q1
        mul256
        vrev64.32 q0, q0
        vrev64.32 q1, q1
        vst1.8  {q0, q1}, [r0]
        bx      r14
ENDFUNC

FUNC(gcm_mulk_256l_arm_crypto)
        // On entry, r0 points to a 256-bit field element A in little-endian
        // words format; r1 points to a field-element K in register format.
        // On exit, A is updated with the product A K.

        vld1.8  {q0, q1}, [r0]
        vld1.8  {q2, q3}, [r1]
        vrev64.8 q0, q0
        vrev64.8 q1, q1
        mul256
        vrev64.8 q0, q0
        vrev64.8 q1, q1
        vst1.8  {q0, q1}, [r0]
        bx      r14
ENDFUNC

///----- That's all, folks --------------------------------------------------