[secnet] / f25519.c

/* -*-c-*-
 *
 * Arithmetic modulo 2^255 - 19
 *
 * (c) 2017 Straylight/Edgeware
 */

/*----- Licensing notice --------------------------------------------------*
 *
 * This file is part of secnet.
 * See README for full list of copyright holders.
 *
 * secnet is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version d of the License, or
 * (at your option) any later version.
 *
 * secnet is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * version 3 along with secnet; if not, see
 * https://www.gnu.org/licenses/gpl.html.
 *
 * This file was originally part of Catacomb, but has been automatically
 * modified for incorporation into secnet: see `import-catacomb-crypto'
 * for details.
 *
 * Catacomb is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Library General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * Catacomb is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with Catacomb; if not, write to the Free
 * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
 * MA 02111-1307, USA.
 */

/*----- Header files ------------------------------------------------------*/

#include "f25519.h"

/*----- Basic setup -------------------------------------------------------*/

typedef f25519_piece piece;

/* Elements x of GF(2^255 - 19) are represented by ten signed integers x_i: x
 * = SUM_{0<=i<10} x_i 2^ceil(51i/2), mostly following Bernstein's original
 * paper.
 */

			typedef  int64  dblpiece;
typedef uint32 upiece;  typedef uint64 udblpiece;
#define P p26
#define PIECEWD(i) ((i)%2 ? 25 : 26)
#define NPIECE 10

#define M26 0x03ffffffu
#define M25 0x01ffffffu
#define B25 0x02000000u
#define B24 0x01000000u

#define PIECES(v) v##0, v##1, v##2, v##3, v##4, v##5, v##6, v##7, v##8, v##9
#define FETCH(v, w) do {						\
  v##0 = (w)->P[0]; v##1 = (w)->P[1];					\
  v##2 = (w)->P[2]; v##3 = (w)->P[3];					\
  v##4 = (w)->P[4]; v##5 = (w)->P[5];					\
  v##6 = (w)->P[6]; v##7 = (w)->P[7];					\
  v##8 = (w)->P[8]; v##9 = (w)->P[9];					\
} while (0)
#define STASH(w, v) do {						\
  (w)->P[0] = v##0; (w)->P[1] = v##1;					\
  (w)->P[2] = v##2; (w)->P[3] = v##3;					\
  (w)->P[4] = v##4; (w)->P[5] = v##5;					\
  (w)->P[6] = v##6; (w)->P[7] = v##7;					\
  (w)->P[8] = v##8; (w)->P[9] = v##9;					\
} while (0)

/*----- Debugging machinery -----------------------------------------------*/

#if defined(F25519_DEBUG)

#include <stdio.h>

#include "mp.h"
#include "mptext.h"

static mp *get_2p255m91(void)
{
  mpw w19 = 19;
  mp *p = MP_NEW, m19;

  p = mp_setbit(p, MP_ZERO, 255);
  mp_build(&m19, &w19, &w19 + 1);
  p = mp_sub(p, p, &m19);
  return (p);
}

DEF_FDUMP(fdump, piece, PIECEWD, NPIECE, 32, get_2p255m91())

#endif

/*----- Loading and storing -----------------------------------------------*/

/* --- @f25519_load@ --- *
 *
 * Arguments:	@f25519 *z@ = where to store the result
 *		@const octet xv[32]@ = source to read
 *
 * Returns:	---
 *
 * Use:		Reads an element of %$\gf{2^{255} - 19}$% in external
 *		representation from @xv@ and stores it in @z@.
 *
 *		External representation is little-endian base-256.  Some
 *		elements have multiple encodings, which are not produced by
 *		correct software; use of noncanonical encodings is not an
 *		error, and toleration of them is considered a performance
 *		feature.
 */

void f25519_load(f25519 *z, const octet xv[32])
{

  uint32 xw0 = LOAD32_L(xv +  0), xw1 = LOAD32_L(xv +  4),
	 xw2 = LOAD32_L(xv +  8), xw3 = LOAD32_L(xv + 12),
	 xw4 = LOAD32_L(xv + 16), xw5 = LOAD32_L(xv + 20),
	 xw6 = LOAD32_L(xv + 24), xw7 = LOAD32_L(xv + 28);
  piece PIECES(x), b, c;

  /* First, split the 32-bit words into the irregularly-sized pieces we need
   * for the field representation.  These pieces are all positive.  We'll do
   * the sign correction afterwards.
   *
   * It may be that the top bit of the input is set.  Avoid trouble by
   * folding that back round into the bottom piece of the representation.
   *
   * Here, we briefly have 0 <= x_0 < 2^26 + 19, but will resolve this later.
   * Otherwise, we have 0 <= x_{2i} < 2^26, and 0 <= x_{2i+1} < 2^25.
   */
  x0 = ((xw0 <<  0)&0x03ffffff) + 19*((xw7 >> 31)&0x00000001);
  x1 = ((xw1 <<  6)&0x01ffffc0) |    ((xw0 >> 26)&0x0000003f);
  x2 = ((xw2 << 13)&0x03ffe000) |    ((xw1 >> 19)&0x00001fff);
  x3 = ((xw3 << 19)&0x01f80000) |    ((xw2 >> 13)&0x0007ffff);
  x4 =				     ((xw3 >>  6)&0x03ffffff);
  x5 =  (xw4 <<  0)&0x01ffffff;
  x6 = ((xw5 <<  7)&0x03ffff80) |    ((xw4 >> 25)&0x0000007f);
  x7 = ((xw6 << 13)&0x01ffe000) |    ((xw5 >> 19)&0x00001fff);
  x8 = ((xw7 << 20)&0x03f00000) |    ((xw6 >> 12)&0x000fffff);
  x9 =				     ((xw7 >>  6)&0x01ffffff);

  /* Next, we convert these pieces into a roughly balanced signed
   * representation.  For each piece, check to see if its top bit is set.  If
   * it is, then lend a bit to the next piece over.  For x_9, this needs to
   * be carried around, which is a little fiddly.  In particular, we delay
   * the carry until after all of the pieces have been balanced.  If we don't
   * do this, then we have to do a more expensive test for nonzeroness to
   * decide whether to lend a bit leftwards rather than just testing a single
   * bit.
   *
   * This fixes up the anomalous size of x_0: the loan of a bit becomes an
   * actual carry if x_0 >= 2^26.  By the end, then, we have:
   *
   *		 { 2^25		if i even
   *	|x_i| <= {
   *		 { 2^24		if i odd
   *
   * Note that we don't try for a canonical representation here: both upper
   * and lower bounds are achievable.
   *
   * All of the x_i at this point are positive, so we don't need to do
   * anything wierd when masking them.
   */
  b = x9&B24; c   = 19&((b >> 19) - (b >> 24)); x9 -= b << 1;
  b = x8&B25; x9 +=      b >> 25;		x8 -= b << 1;
  b = x7&B24; x8 +=	 b >> 24;		x7 -= b << 1;
  b = x6&B25; x7 +=	 b >> 25;		x6 -= b << 1;
  b = x5&B24; x6 +=	 b >> 24;		x5 -= b << 1;
  b = x4&B25; x5 +=	 b >> 25;		x4 -= b << 1;
  b = x3&B24; x4 +=	 b >> 24;		x3 -= b << 1;
  b = x2&B25; x3 +=	 b >> 25;		x2 -= b << 1;
  b = x1&B24; x2 +=	 b >> 24;		x1 -= b << 1;
  b = x0&B25; x1 +=     (b >> 25) + (x0 >> 26); x0 = (x0&M26) - (b << 1);
	      x0 +=	 c;

  /* And with that, we're done. */
  STASH(z, x);
}

/* --- @f25519_store@ --- *
 *
 * Arguments:	@octet zv[32]@ = where to write the result
 *		@const f25519 *x@ = the field element to write
 *
 * Returns:	---
 *
 * Use:		Stores a field element in the given octet vector in external
 *		representation.  A canonical encoding is always stored, so,
 *		in particular, the top bit of @xv[31]@ is always left clear.
 */

void f25519_store(octet zv[32], const f25519 *x)
{

  piece PIECES(x), PIECES(y), c, d;
  uint32 zw0, zw1, zw2, zw3, zw4, zw5, zw6, zw7;
  mask32 m;

  FETCH(x, x);

  /* First, propagate the carries throughout the pieces.  By the end of this,
   * we'll have all of the pieces canonically sized and positive, and maybe
   * there'll be (signed) carry out.  The carry c is in { -1, 0, +1 }, and
   * the remaining value will be in the half-open interval [0, 2^255).  The
   * whole represented value is then x + 2^255 c.
   *
   * It's worth paying careful attention to the bounds.  We assume that we
   * start out with |x_i| <= 2^30.  We start by cutting off and reducing the
   * carry c_9 from the topmost piece, x_9.  This leaves 0 <= x_9 < 2^25; and
   * we'll have |c_9| <= 2^5.  We multiply this by 19 and we'll add it onto
   * x_0 and propagate the carries: but what bounds can we calculate on x
   * before this?
   *
   * Let o_i = floor(51 i/2).  We have X_i = SUM_{0<=j<i} x_j 2^{o_i}, so
   * x = X_10.  We see, inductively, that |X_i| < 2^{31+o_{i-1}}: X_0 = 0;
   * |x_i| <= 2^30; and |X_{i+1}| = |X_i + x_i 2^{o_i}| <= |X_i| + 2^{30+o_i}
   * < 2^{31+o_i}.  Then x = X_9 + 2^230 x_9, and we have better bounds for
   * x_9, so
   *
   *	-2^235 < x + 19 c_9 < 2^255 + 2^235
   *
   * Here, the x_i are signed, so we must be cautious about bithacking them.
   */
	      c = ASR(piece, x9, 25); x9 = (upiece)x9&M25;
  x0 += 19*c; c = ASR(piece, x0, 26); x0 = (upiece)x0&M26;
  x1 +=    c; c = ASR(piece, x1, 25); x1 = (upiece)x1&M25;
  x2 +=    c; c = ASR(piece, x2, 26); x2 = (upiece)x2&M26;
  x3 +=    c; c = ASR(piece, x3, 25); x3 = (upiece)x3&M25;
  x4 +=    c; c = ASR(piece, x4, 26); x4 = (upiece)x4&M26;
  x5 +=    c; c = ASR(piece, x5, 25); x5 = (upiece)x5&M25;
  x6 +=    c; c = ASR(piece, x6, 26); x6 = (upiece)x6&M26;
  x7 +=    c; c = ASR(piece, x7, 25); x7 = (upiece)x7&M25;
  x8 +=    c; c = ASR(piece, x8, 26); x8 = (upiece)x8&M26;
  x9 +=    c; c = ASR(piece, x9, 25); x9 = (upiece)x9&M25;

  /* Now we have a slightly fiddly job to do.  If c = +1, or if c = 0 and
   * x >= 2^255 - 19, then we should subtract 2^255 - 19 from the whole
   * value; if c = -1 then we should add 2^255 - 19; and otherwise we should
   * do nothing.
   *
   * But conditional behaviour is bad, m'kay.  So here's what we do instead.
   *
   * The first job is to sort out what we wanted to do.  If c = -1 then we
   * want to (a) invert the constant addend and (b) feed in a carry-in;
   * otherwise, we don't.
   */
  m = SIGN(c);
  d = m&1;

  /* Now do the addition/subtraction.  Remember that all of the x_i are
   * nonnegative, so shifting and masking are safe and easy.
   */
  d += x0 + (19 ^ (M26&m)); y0 = d&M26; d >>= 26;
  d += x1 +	  (M25&m);  y1 = d&M25; d >>= 25;
  d += x2 +	  (M26&m);  y2 = d&M26; d >>= 26;
  d += x3 +	  (M25&m);  y3 = d&M25; d >>= 25;
  d += x4 +	  (M26&m);  y4 = d&M26; d >>= 26;
  d += x5 +	  (M25&m);  y5 = d&M25; d >>= 25;
  d += x6 +	  (M26&m);  y6 = d&M26; d >>= 26;
  d += x7 +	  (M25&m);  y7 = d&M25; d >>= 25;
  d += x8 +	  (M26&m);  y8 = d&M26; d >>= 26;
  d += x9 +	  (M25&m);  y9 = d&M25; d >>= 25;

  /* The final carry-out is in d; since we only did addition, and the x_i are
   * nonnegative, then d is in { 0, 1 }.  We want to keep y, rather than x,
   * if (a) c /= 0 (in which case we know that the old value was
   * unsatisfactory), or (b) if d = 1 (in which case, if c = 0, we know that
   * the subtraction didn't cause a borrow, so we must be in the case where
   * 2^255 - 19 <= x < 2^255).
   */
  m = NONZEROP(c) | ~NONZEROP(d - 1);
  x0 = (y0&m) | (x0&~m); x1 = (y1&m) | (x1&~m);
  x2 = (y2&m) | (x2&~m); x3 = (y3&m) | (x3&~m);
  x4 = (y4&m) | (x4&~m); x5 = (y5&m) | (x5&~m);
  x6 = (y6&m) | (x6&~m); x7 = (y7&m) | (x7&~m);
  x8 = (y8&m) | (x8&~m); x9 = (y9&m) | (x9&~m);

  /* Extract 32-bit words from the value. */
  zw0 = ((x0 >>  0)&0x03ffffff) | (((uint32)x1 << 26)&0xfc000000);
  zw1 = ((x1 >>  6)&0x0007ffff) | (((uint32)x2 << 19)&0xfff80000);
  zw2 = ((x2 >> 13)&0x00001fff) | (((uint32)x3 << 13)&0xffffe000);
  zw3 = ((x3 >> 19)&0x0000003f) | (((uint32)x4 <<  6)&0xffffffc0);
  zw4 = ((x5 >>  0)&0x01ffffff) | (((uint32)x6 << 25)&0xfe000000);
  zw5 = ((x6 >>  7)&0x0007ffff) | (((uint32)x7 << 19)&0xfff80000);
  zw6 = ((x7 >> 13)&0x00000fff) | (((uint32)x8 << 12)&0xfffff000);
  zw7 = ((x8 >> 20)&0x0000003f) | (((uint32)x9 <<  6)&0x7fffffc0);

  /* Store the result as an octet string. */
  STORE32_L(zv +  0, zw0); STORE32_L(zv +  4, zw1);
  STORE32_L(zv +  8, zw2); STORE32_L(zv + 12, zw3);
  STORE32_L(zv + 16, zw4); STORE32_L(zv + 20, zw5);
  STORE32_L(zv + 24, zw6); STORE32_L(zv + 28, zw7);
}

/* --- @f25519_set@ --- *
 *
 * Arguments:	@f25519 *z@ = where to write the result
 *		@int a@ = a small-ish constant
 *
 * Returns:	---
 *
 * Use:		Sets @z@ to equal @a@.
 */

void f25519_set(f25519 *x, int a)
{
  unsigned i;

  x->P[0] = a;
  for (i = 1; i < NPIECE; i++) x->P[i] = 0;
}

/*----- Basic arithmetic --------------------------------------------------*/

/* --- @f25519_add@ --- *
 *
 * Arguments:	@f25519 *z@ = where to put the result (may alias @x@ or @y@)
 *		@const f25519 *x, *y@ = two operands
 *
 * Returns:	---
 *
 * Use:		Set @z@ to the sum %$x + y$%.
 */

void f25519_add(f25519 *z, const f25519 *x, const f25519 *y)
{
  z->P[0] = x->P[0] + y->P[0]; z->P[1] = x->P[1] + y->P[1];
  z->P[2] = x->P[2] + y->P[2]; z->P[3] = x->P[3] + y->P[3];
  z->P[4] = x->P[4] + y->P[4]; z->P[5] = x->P[5] + y->P[5];
  z->P[6] = x->P[6] + y->P[6]; z->P[7] = x->P[7] + y->P[7];
  z->P[8] = x->P[8] + y->P[8]; z->P[9] = x->P[9] + y->P[9];
}

/* --- @f25519_sub@ --- *
 *
 * Arguments:	@f25519 *z@ = where to put the result (may alias @x@ or @y@)
 *		@const f25519 *x, *y@ = two operands
 *
 * Returns:	---
 *
 * Use:		Set @z@ to the difference %$x - y$%.
 */

void f25519_sub(f25519 *z, const f25519 *x, const f25519 *y)
{
  z->P[0] = x->P[0] - y->P[0]; z->P[1] = x->P[1] - y->P[1];
  z->P[2] = x->P[2] - y->P[2]; z->P[3] = x->P[3] - y->P[3];
  z->P[4] = x->P[4] - y->P[4]; z->P[5] = x->P[5] - y->P[5];
  z->P[6] = x->P[6] - y->P[6]; z->P[7] = x->P[7] - y->P[7];
  z->P[8] = x->P[8] - y->P[8]; z->P[9] = x->P[9] - y->P[9];
}

/* --- @f25519_neg@ --- *
 *
 * Arguments:	@f25519 *z@ = where to put the result (may alias @x@)
 *		@const f25519 *x@ = an operand
 *
 * Returns:	---
 *
 * Use:		Set @z = -x@.
 */

void f25519_neg(f25519 *z, const f25519 *x)
{
  z->P[0] = -x->P[0]; z->P[1] = -x->P[1];
  z->P[2] = -x->P[2]; z->P[3] = -x->P[3];
  z->P[4] = -x->P[4]; z->P[5] = -x->P[5];
  z->P[6] = -x->P[6]; z->P[7] = -x->P[7];
  z->P[8] = -x->P[8]; z->P[9] = -x->P[9];
}

/*----- Constant-time utilities -------------------------------------------*/

/* --- @f25519_pick2@ --- *
 *
 * Arguments:	@f25519 *z@ = where to put the result (may alias @x@ or @y@)
 *		@const f25519 *x, *y@ = two operands
 *		@uint32 m@ = a mask
 *
 * Returns:	---
 *
 * Use:		If @m@ is zero, set @z = y@; if @m@ is all-bits-set, then set
 *		@z = x@.  If @m@ has some other value, then scramble @z@ in
 *		an unhelpful way.
 */

void f25519_pick2(f25519 *z, const f25519 *x, const f25519 *y, uint32 m)
{
  mask32 mm = FIX_MASK32(m);

  z->P[0] = PICK2(x->P[0], y->P[0], mm);
  z->P[1] = PICK2(x->P[1], y->P[1], mm);
  z->P[2] = PICK2(x->P[2], y->P[2], mm);
  z->P[3] = PICK2(x->P[3], y->P[3], mm);
  z->P[4] = PICK2(x->P[4], y->P[4], mm);
  z->P[5] = PICK2(x->P[5], y->P[5], mm);
  z->P[6] = PICK2(x->P[6], y->P[6], mm);
  z->P[7] = PICK2(x->P[7], y->P[7], mm);
  z->P[8] = PICK2(x->P[8], y->P[8], mm);
  z->P[9] = PICK2(x->P[9], y->P[9], mm);
}

/* --- @f25519_pickn@ --- *
 *
 * Arguments:	@f25519 *z@ = where to put the result
 *		@const f25519 *v@ = a table of entries
 *		@size_t n@ = the number of entries in @v@
 *		@size_t i@ = an index
 *
 * Returns:	---
 *
 * Use:		If @0 <= i < n < 32@ then set @z = v[i]@.  If @n >= 32@ then
 *		do something unhelpful; otherwise, if @i >= n@ then set @z@
 *		to zero.
 */

void f25519_pickn(f25519 *z, const f25519 *v, size_t n, size_t i)
{
  uint32 b = (uint32)1 << (31 - i);
  mask32 m;

  z->P[0] = z->P[1] = z->P[2] = z->P[3] = z->P[4] =
    z->P[5] = z->P[6] = z->P[7] = z->P[8] = z->P[9] = 0;
  while (n--) {
    m = SIGN(b);
    CONDPICK(z->P[0], v->P[0], m);
    CONDPICK(z->P[1], v->P[1], m);
    CONDPICK(z->P[2], v->P[2], m);
    CONDPICK(z->P[3], v->P[3], m);
    CONDPICK(z->P[4], v->P[4], m);
    CONDPICK(z->P[5], v->P[5], m);
    CONDPICK(z->P[6], v->P[6], m);
    CONDPICK(z->P[7], v->P[7], m);
    CONDPICK(z->P[8], v->P[8], m);
    CONDPICK(z->P[9], v->P[9], m);
    v++; b <<= 1;
  }
}

/* --- @f25519_condswap@ --- *
 *
 * Arguments:	@f25519 *x, *y@ = two operands
 *		@uint32 m@ = a mask
 *
 * Returns:	---
 *
 * Use:		If @m@ is zero, do nothing; if @m@ is all-bits-set, then
 *		exchange @x@ and @y@.  If @m@ has some other value, then
 *		scramble @x@ and @y@ in an unhelpful way.
 */

void f25519_condswap(f25519 *x, f25519 *y, uint32 m)
{
  mask32 mm = FIX_MASK32(m);

  CONDSWAP(x->P[0], y->P[0], mm);
  CONDSWAP(x->P[1], y->P[1], mm);
  CONDSWAP(x->P[2], y->P[2], mm);
  CONDSWAP(x->P[3], y->P[3], mm);
  CONDSWAP(x->P[4], y->P[4], mm);
  CONDSWAP(x->P[5], y->P[5], mm);
  CONDSWAP(x->P[6], y->P[6], mm);
  CONDSWAP(x->P[7], y->P[7], mm);
  CONDSWAP(x->P[8], y->P[8], mm);
  CONDSWAP(x->P[9], y->P[9], mm);
}

/* --- @f25519_condneg@ --- *
 *
 * Arguments:	@f25519 *z@ = where to put the result (may alias @x@)
 *		@const f25519 *x@ = an operand
 *		@uint32 m@ = a mask
 *
 * Returns:	---
 *
 * Use:		If @m@ is zero, set @z = x@; if @m@ is all-bits-set, then set
 *		@z = -x@.  If @m@ has some other value then scramble @z@ in
 *		an unhelpful way.
 */

void f25519_condneg(f25519 *z, const f25519 *x, uint32 m)
{
  mask32 m_xor = FIX_MASK32(m);
  piece m_add = m&1;
# define CONDNEG(x) (((x) ^ m_xor) + m_add)

  z->P[0] = CONDNEG(x->P[0]);
  z->P[1] = CONDNEG(x->P[1]);
  z->P[2] = CONDNEG(x->P[2]);
  z->P[3] = CONDNEG(x->P[3]);
  z->P[4] = CONDNEG(x->P[4]);
  z->P[5] = CONDNEG(x->P[5]);
  z->P[6] = CONDNEG(x->P[6]);
  z->P[7] = CONDNEG(x->P[7]);
  z->P[8] = CONDNEG(x->P[8]);
  z->P[9] = CONDNEG(x->P[9]);

#undef CONDNEG
}

/*----- Multiplication ----------------------------------------------------*/

/* Let B = 2^63 - 1 be the largest value such that +B and -B can be
 * represented in a double-precision piece.  On entry, it must be the case
 * that |X_i| <= M <= B - 2^25 for some M.  If this is the case, then, on
 * exit, we will have |Z_i| <= 2^25 + 19 M/2^25.
 */
#define CARRYSTEP(z, x, m, b, f, xx, n) do {				\
  (z) = (dblpiece)((udblpiece)(x)&(m)) - (b) +				\
    (f)*ASR(dblpiece, (xx), (n));					\
} while (0)
#define CARRY_REDUCE(z, x) do {						\
  dblpiece PIECES(_t);							\
									\
  /* Bias the input pieces.  This keeps the carries and so on centred	\
   * around zero rather than biased positive.				\
   */									\
  _t0 = (x##0) + B25; _t1 = (x##1) + B24;				\
  _t2 = (x##2) + B25; _t3 = (x##3) + B24;				\
  _t4 = (x##4) + B25; _t5 = (x##5) + B24;				\
  _t6 = (x##6) + B25; _t7 = (x##7) + B24;				\
  _t8 = (x##8) + B25; _t9 = (x##9) + B24;				\
									\
  /* Calculate the reduced pieces.  Careful with the bithacking. */	\
  CARRYSTEP(z##0, _t0, M26, B25, 19, _t9, 25);				\
  CARRYSTEP(z##1, _t1, M25, B24,  1, _t0, 26);				\
  CARRYSTEP(z##2, _t2, M26, B25,  1, _t1, 25);				\
  CARRYSTEP(z##3, _t3, M25, B24,  1, _t2, 26);				\
  CARRYSTEP(z##4, _t4, M26, B25,  1, _t3, 25);				\
  CARRYSTEP(z##5, _t5, M25, B24,  1, _t4, 26);				\
  CARRYSTEP(z##6, _t6, M26, B25,  1, _t5, 25);				\
  CARRYSTEP(z##7, _t7, M25, B24,  1, _t6, 26);				\
  CARRYSTEP(z##8, _t8, M26, B25,  1, _t7, 25);				\
  CARRYSTEP(z##9, _t9, M25, B24,  1, _t8, 26);				\
} while (0)

/* --- @f25519_mulconst@ --- *
 *
 * Arguments:	@f25519 *z@ = where to put the result (may alias @x@)
 *		@const f25519 *x@ = an operand
 *		@long a@ = a small-ish constant; %$|a| < 2^{20}$%.
 *
 * Returns:	---
 *
 * Use:		Set @z@ to the product %$a x$%.
 */

void f25519_mulconst(f25519 *z, const f25519 *x, long a)
{

  piece PIECES(x);
  dblpiece PIECES(z), aa = a;

  FETCH(x, x);

  /* Suppose that |x_i| <= 2^27, and |a| <= 2^23.  Then we'll have
   * |z_i| <= 2^50.
   */
  z0 = aa*x0; z1 = aa*x1; z2 = aa*x2; z3 = aa*x3; z4 = aa*x4;
  z5 = aa*x5; z6 = aa*x6; z7 = aa*x7; z8 = aa*x8; z9 = aa*x9;

  /* Following `CARRY_REDUCE', we'll have |z_i| <= 2^26. */
  CARRY_REDUCE(z, z);
  STASH(z, z);
}

/* --- @f25519_mul@ --- *
 *
 * Arguments:	@f25519 *z@ = where to put the result (may alias @x@ or @y@)
 *		@const f25519 *x, *y@ = two operands
 *
 * Returns:	---
 *
 * Use:		Set @z@ to the product %$x y$%.
 */

void f25519_mul(f25519 *z, const f25519 *x, const f25519 *y)
{

  piece PIECES(x), PIECES(y);
  dblpiece PIECES(z);
  unsigned i;

  FETCH(x, x); FETCH(y, y);

  /* Suppose that |x_i|, |y_i| <= 2^27.  Then we'll have
   *
   *	|z_0| <= 267*2^54
   *	|z_1| <= 154*2^54
   *	|z_2| <= 213*2^54
   *	|z_3| <= 118*2^54
   *	|z_4| <= 159*2^54
   *	|z_5| <=  82*2^54
   *	|z_6| <= 105*2^54
   *	|z_7| <=  46*2^54
   *	|z_8| <=  51*2^54
   *	|z_9| <=  10*2^54
   *
   * all of which are less than 2^63 - 2^25.
   */

#define M(a, b) ((dblpiece)(a)*(b))
  z0 =	   M(x0, y0) +
       19*(M(x2, y8) + M(x4, y6) + M(x6, y4) + M(x8, y2)) +
       38*(M(x1, y9) + M(x3, y7) + M(x5, y5) + M(x7, y3) + M(x9, y1));
  z1 =	   M(x0, y1) + M(x1, y0) +
       19*(M(x2, y9) + M(x3, y8) + M(x4, y7) + M(x5, y6) +
	   M(x6, y5) + M(x7, y4) + M(x8, y3) + M(x9, y2));
  z2 =	   M(x0, y2) + M(x2, y0) +
	2* M(x1, y1) +
       19*(M(x4, y8) + M(x6, y6) + M(x8, y4)) +
       38*(M(x3, y9) + M(x5, y7) + M(x7, y5) + M(x9, y3));
  z3 =	   M(x0, y3) + M(x1, y2) + M(x2, y1) + M(x3, y0) +
       19*(M(x4, y9) + M(x5, y8) + M(x6, y7) +
	   M(x7, y6) + M(x8, y5) + M(x9, y4));
  z4 =	   M(x0, y4) + M(x2, y2) + M(x4, y0) +
	2*(M(x1, y3) + M(x3, y1)) +
       19*(M(x6, y8) + M(x8, y6)) +
       38*(M(x5, y9) + M(x7, y7) + M(x9, y5));
  z5 =	   M(x0, y5) + M(x1, y4) + M(x2, y3) +
	   M(x3, y2) + M(x4, y1) + M(x5, y0) +
       19*(M(x6, y9) + M(x7, y8) + M(x8, y7) + M(x9, y6));
  z6 =	   M(x0, y6) + M(x2, y4) + M(x4, y2) + M(x6, y0) +
	2*(M(x1, y5) + M(x3, y3) + M(x5, y1)) +
       19* M(x8, y8) +
       38*(M(x7, y9) + M(x9, y7));
  z7 =	   M(x0, y7) + M(x1, y6) + M(x2, y5) + M(x3, y4) +
	   M(x4, y3) + M(x5, y2) + M(x6, y1) + M(x7, y0) +
       19*(M(x8, y9) + M(x9, y8));
  z8 =	   M(x0, y8) + M(x2, y6) + M(x4, y4) + M(x6, y2) + M(x8, y0) +
	2*(M(x1, y7) + M(x3, y5) + M(x5, y3) + M(x7, y1)) +
       38* M(x9, y9);
  z9 =	   M(x0, y9) + M(x1, y8) + M(x2, y7) + M(x3, y6) + M(x4, y5) +
	   M(x5, y4) + M(x6, y3) + M(x7, y2) + M(x8, y1) + M(x9, y0);
#undef M

  /* From above, we have |z_i| <= 2^63 - 2^25.  A pass of `CARRY_REDUCE' will
   * leave |z_i| <= 2^38 + 2^25; and a second pass will leave |z_i| <= 2^25 +
   * 2^13, which is comfortable for an addition prior to the next
   * multiplication.
   */
  for (i = 0; i < 2; i++) CARRY_REDUCE(z, z);
  STASH(z, z);
}

/* --- @f25519_sqr@ --- *
 *
 * Arguments:	@f25519 *z@ = where to put the result (may alias @x@ or @y@)
 *		@const f25519 *x@ = an operand
 *
 * Returns:	---
 *
 * Use:		Set @z@ to the square %$x^2$%.
 */

void f25519_sqr(f25519 *z, const f25519 *x)
{

  piece PIECES(x);
  dblpiece PIECES(z);
  unsigned i;

  FETCH(x, x);

  /* See `f25519_mul' for bounds. */

#define M(a, b) ((dblpiece)(a)*(b))
  z0 =	   M(x0, x0) +
       38*(M(x2, x8) + M(x4, x6) + M(x5, x5)) +
       76*(M(x1, x9) + M(x3, x7));
  z1 =  2* M(x0, x1) +
       38*(M(x2, x9) + M(x3, x8) + M(x4, x7) + M(x5, x6));
  z2 =	2*(M(x0, x2) + M(x1, x1)) +
       19* M(x6, x6) +
       38* M(x4, x8) +
       76*(M(x3, x9) + M(x5, x7));
  z3 =  2*(M(x0, x3) + M(x1, x2)) +
       38*(M(x4, x9) + M(x5, x8) + M(x6, x7));
  z4 =	   M(x2, x2) +
	2* M(x0, x4) +
	4* M(x1, x3) +
       38*(M(x6, x8) + M(x7, x7)) +
       76* M(x5, x9);
  z5 =	2*(M(x0, x5) + M(x1, x4) + M(x2, x3)) +
       38*(M(x6, x9) + M(x7, x8));
  z6 =	2*(M(x0, x6) + M(x2, x4) + M(x3, x3)) +
	4* M(x1, x5) +
       19* M(x8, x8) +
       76* M(x7, x9);
  z7 =	2*(M(x0, x7) + M(x1, x6) + M(x2, x5) + M(x3, x4)) +
       38* M(x8, x9);
  z8 =	   M(x4, x4) +
	2*(M(x0, x8) + M(x2, x6)) +
	4*(M(x1, x7) + M(x3, x5)) +
       38* M(x9, x9);
  z9 =	2*(M(x0, x9) + M(x1, x8) + M(x2, x7) + M(x3, x6) + M(x4, x5));
#undef M

  /* See `f25519_mul' for details. */
  for (i = 0; i < 2; i++) CARRY_REDUCE(z, z);
  STASH(z, z);
}

/*----- More complicated things -------------------------------------------*/

/* --- @f25519_inv@ --- *
 *
 * Arguments:	@f25519 *z@ = where to put the result (may alias @x@)
 *		@const f25519 *x@ = an operand
 *
 * Returns:	---
 *
 * Use:		Stores in @z@ the multiplicative inverse %$x^{-1}$%.  If
 *		%$x = 0$% then @z@ is set to zero.  This is considered a
 *		feature.
 */

void f25519_inv(f25519 *z, const f25519 *x)
{
  f25519 t, u, t2, t11, t2p10m1, t2p50m1;
  unsigned i;

#define SQRN(z, x, n) do {						\
  f25519_sqr((z), (x));							\
  for (i = 1; i < (n); i++) f25519_sqr((z), (z));			\
} while (0)

  /* Calculate x^-1 = x^(p - 2) = x^(2^255 - 21), which also handles x = 0 as
   * intended.  The addition chain here is from Bernstein's implementation; I
   * couldn't find a better one.
   */					/* step | value */
  f25519_sqr(&t2, x);			/*    1 | 2 */
  SQRN(&u, &t2, 2);			/*    3 | 8 */
  f25519_mul(&t, &u, x);		/*    4 | 9 */
  f25519_mul(&t11, &t, &t2);		/*    5 | 11 = 2^5 - 21 */
  f25519_sqr(&u, &t11);			/*    6 | 22 */
  f25519_mul(&t, &t, &u);		/*    7 | 31 = 2^5 - 1 */
  SQRN(&u, &t, 5);			/*   12 | 2^10 - 2^5 */
  f25519_mul(&t2p10m1, &t, &u);		/*   13 | 2^10 - 1 */
  SQRN(&u, &t2p10m1, 10);		/*   23 | 2^20 - 2^10 */
  f25519_mul(&t, &t2p10m1, &u);		/*   24 | 2^20 - 1 */
  SQRN(&u, &t, 20);			/*   44 | 2^40 - 2^20 */
  f25519_mul(&t, &t, &u);		/*   45 | 2^40 - 1 */
  SQRN(&u, &t, 10);			/*   55 | 2^50 - 2^10 */
  f25519_mul(&t2p50m1, &t2p10m1, &u);	/*   56 | 2^50 - 1 */
  SQRN(&u, &t2p50m1, 50);		/*  106 | 2^100 - 2^50 */
  f25519_mul(&t, &t2p50m1, &u);		/*  107 | 2^100 - 1 */
  SQRN(&u, &t, 100);			/*  207 | 2^200 - 2^100 */
  f25519_mul(&t, &t, &u);		/*  208 | 2^200 - 1 */
  SQRN(&u, &t, 50);			/*  258 | 2^250 - 2^50 */
  f25519_mul(&t, &t2p50m1, &u);		/*  259 | 2^250 - 1 */
  SQRN(&u, &t, 5);			/*  264 | 2^255 - 2^5 */
  f25519_mul(z, &u, &t11);		/*  265 | 2^255 - 21 */

#undef SQRN
}

/* --- @f25519_quosqrt@ --- *
 *
 * Arguments:	@f25519 *z@ = where to put the result (may alias @x@ or @y@)
 *		@const f25519 *x, *y@ = two operands
 *
 * Returns:	Zero if successful, @-1@ if %$x/y$% is not a square.
 *
 * Use:		Stores in @z@ the one of the square roots %$\pm\sqrt{x/y}$%.
 *		If %$x = y = 0% then the result is zero; if %$y = 0$% but %$x
 *		\ne 0$% then the operation fails.  If you wanted a specific
 *		square root then you'll have to pick it yourself.
 */

static const piece sqrtm1_pieces[NPIECE] = {
  -32595792,  -7943725,   9377950,   3500415,  12389472,
    -272473, -25146209,  -2005654,    326686,  11406482
};
#define SQRTM1 ((const f25519 *)sqrtm1_pieces)

int f25519_quosqrt(f25519 *z, const f25519 *x, const f25519 *y)
{
  f25519 t, u, v, w, t15;
  octet xb[32], b0[32], b1[32];
  int32 rc = -1;
  mask32 m;
  unsigned i;

#define SQRN(z, x, n) do {						\
  f25519_sqr((z), (x));							\
  for (i = 1; i < (n); i++) f25519_sqr((z), (z));			\
} while (0)

  /* This is a bit tricky; the algorithm is loosely based on Bernstein, Duif,
   * Lange, Schwabe, and Yang, `High-speed high-security signatures',
   * 2011-09-26, https://ed25519.cr.yp.to/ed25519-20110926.pdf.
   */
  f25519_mul(&v, x, y);

  /* Now for an addition chain. */	/* step | value */
  f25519_sqr(&u, &v);			/*    1 | 2 */
  f25519_mul(&t, &u, &v);		/*    2 | 3 */
  SQRN(&u, &t, 2);			/*    4 | 12 */
  f25519_mul(&t15, &u, &t);		/*    5 | 15 */
  f25519_sqr(&u, &t15);			/*    6 | 30 */
  f25519_mul(&t, &u, &v);		/*    7 | 31 = 2^5 - 1 */
  SQRN(&u, &t, 5);			/*   12 | 2^10 - 2^5 */
  f25519_mul(&t, &u, &t);		/*   13 | 2^10 - 1 */
  SQRN(&u, &t, 10);			/*   23 | 2^20 - 2^10 */
  f25519_mul(&u, &u, &t);		/*   24 | 2^20 - 1 */
  SQRN(&u, &u, 10);			/*   34 | 2^30 - 2^10 */
  f25519_mul(&t, &u, &t);		/*   35 | 2^30 - 1 */
  f25519_sqr(&u, &t);			/*   36 | 2^31 - 2 */
  f25519_mul(&t, &u, &v);		/*   37 | 2^31 - 1 */
  SQRN(&u, &t, 31);			/*   68 | 2^62 - 2^31 */
  f25519_mul(&t, &u, &t);		/*   69 | 2^62 - 1 */
  SQRN(&u, &t, 62);			/*  131 | 2^124 - 2^62 */
  f25519_mul(&t, &u, &t);		/*  132 | 2^124 - 1 */
  SQRN(&u, &t, 124);			/*  256 | 2^248 - 2^124 */
  f25519_mul(&t, &u, &t);		/*  257 | 2^248 - 1 */
  f25519_sqr(&u, &t);			/*  258 | 2^249 - 2 */
  f25519_mul(&t, &u, &v);		/*  259 | 2^249 - 1 */
  SQRN(&t, &t, 3);			/*  262 | 2^252 - 8 */
  f25519_sqr(&u, &t);			/*  263 | 2^253 - 16 */
  f25519_mul(&t, &u, &t);		/*  264 | 3*2^252 - 24 */
  f25519_mul(&t, &t, &t15);		/*  265 | 3*2^252 - 9 */
  f25519_mul(&w, &t, &v);		/*  266 | 3*2^252 - 8 */

  /* Awesome.  Now let me explain.  Let v be a square in GF(p), and let w =
   * v^(3*2^252 - 8).  In particular, let's consider
   *
   *    v^2 w^4 = v^2 v^{3*2^254 - 32} = (v^{2^254 - 10})^3
   *
   * But 2^254 - 10 = ((2^255 - 19) - 1)/2 = (p - 1)/2.  Since v is a square,
   * it has order dividing (p - 1)/2, and therefore v^2 w^4 = 1 and
   *
   *    w^4 = 1/v^2
   *
   * That in turn implies that w^2 = ±1/v.  Now, recall that v = x y, and let
   * w' = w x.  Then w'^2 = ±x^2/v = ±x/y.  If y w'^2 = x then we set
   * z = w', since we have z^2 = x/y; otherwise let z = i w', where i^2 = -1,
   * so z^2 = -w^2 = x/y, and we're done.
   *
   * The easiest way to compare is to encode.  This isn't as wasteful as it
   * sounds: the hard part is normalizing the representations, which we have
   * to do anyway.
   */
  f25519_mul(&w, &w, x);
  f25519_sqr(&t, &w);
  f25519_mul(&t, &t, y);
  f25519_neg(&u, &t);
  f25519_store(xb, x);
  f25519_store(b0, &t);
  f25519_store(b1, &u);
  f25519_mul(&u, &w, SQRTM1);

  m = -consttime_memeq(b0, xb, 32);
  rc = PICK2(0, rc, m);
  f25519_pick2(z, &w, &u, m);
  m = -consttime_memeq(b1, xb, 32);
  rc = PICK2(0, rc, m);

  /* And we're done. */
  return (rc);
}

/*----- That's all, folks -------------------------------------------------*/