[catacomb] / math / f25519.c

/* -*-c-*-
 *
 * Arithmetic modulo 2^255 - 19
 *
 * (c) 2017 Straylight/Edgeware
 */

/*----- Licensing notice --------------------------------------------------*
 *
 * This file is part of Catacomb.
 *
 * Catacomb is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Library General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * Catacomb is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with Catacomb; if not, write to the Free
 * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
 * MA 02111-1307, USA.
 */

/*----- Header files ------------------------------------------------------*/

#include "config.h"

#include "f25519.h"

/*----- Basic setup -------------------------------------------------------*/

#if F25519_IMPL == 26
/* Elements x of GF(2^255 - 19) are represented by ten signed integers x_i: x
 * = SUM_{0<=i<10} x_i 2^ceil(51i/2), mostly following Bernstein's original
 * paper.
 */

typedef  int32  piece;  typedef  int64  dblpiece;
typedef uint32 upiece;  typedef uint64 udblpiece;
#define P p26
#define PIECEWD(i) ((i)%2 ? 25 : 26)
#define NPIECE 10

#define M26 0x03ffffffu
#define M25 0x01ffffffu
#define B26 0x04000000u
#define B25 0x02000000u
#define B24 0x01000000u

#define PIECES(v) v##0, v##1, v##2, v##3, v##4, v##5, v##6, v##7, v##8, v##9
#define FETCH(v, w) do {                                                \
  v##0 = (w)->P[0]; v##1 = (w)->P[1];                                   \
  v##2 = (w)->P[2]; v##3 = (w)->P[3];                                   \
  v##4 = (w)->P[4]; v##5 = (w)->P[5];                                   \
  v##6 = (w)->P[6]; v##7 = (w)->P[7];                                   \
  v##8 = (w)->P[8]; v##9 = (w)->P[9];                                   \
} while (0)
#define STASH(w, v) do {                                                \
  (w)->P[0] = v##0; (w)->P[1] = v##1;                                   \
  (w)->P[2] = v##2; (w)->P[3] = v##3;                                   \
  (w)->P[4] = v##4; (w)->P[5] = v##5;                                   \
  (w)->P[6] = v##6; (w)->P[7] = v##7;                                   \
  (w)->P[8] = v##8; (w)->P[9] = v##9;                                   \
} while (0)

#elif F25519_IMPL == 10
/* Elements x of GF(2^255 - 19) are represented by 26 signed integers x_i: x
 * = SUM_{0<=i<26} x_i 2^ceil(255i/26); i.e., most pieces are 10 bits wide,
 * except for pieces 5, 10, 15, 20, and 25 which have 9 bits.
 */

typedef  int16  piece;  typedef  int32  dblpiece;
typedef uint16 upiece;  typedef uint32 udblpiece;
#define P p10
#define PIECEWD(i)                                                      \
    ((i) == 5 || (i) == 10 || (i) == 15 || (i) == 20 || (i) == 25 ? 9 : 10)
#define NPIECE 26

#define B10 0x0400
#define B9 0x200
#define B8 0x100
#define M10 0x3ff
#define M9 0x1ff

#endif

/*----- Debugging machinery -----------------------------------------------*/

#if defined(F25519_DEBUG) || defined(TEST_RIG)

#include <stdio.h>

#include "mp.h"
#include "mptext.h"

static mp *get_2p255m91(void)
{
  mpw w19 = 19;
  mp *p = MP_NEW, m19;

  p = mp_setbit(p, MP_ZERO, 255);
  mp_build(&m19, &w19, &w19 + 1);
  p = mp_sub(p, p, &m19);
  return (p);
}

DEF_FDUMP(fdump, piece, PIECEWD, NPIECE, 32, get_2p255m91())

#endif

/*----- Loading and storing -----------------------------------------------*/

/* --- @f25519_load@ --- *
 *
 * Arguments:   @f25519 *z@ = where to store the result
 *              @const octet xv[32]@ = source to read
 *
 * Returns:     ---
 *
 * Use:         Reads an element of %$\gf{2^{255} - 19}$% in external
 *              representation from @xv@ and stores it in @z@.
 *
 *              External representation is little-endian base-256.  Some
 *              elements have multiple encodings, which are not produced by
 *              correct software; use of noncanonical encodings is not an
 *              error, and toleration of them is considered a performance
 *              feature.
 */

void f25519_load(f25519 *z, const octet xv[32])
{
#if F25519_IMPL == 26

  uint32 xw0 = LOAD32_L(xv +  0), xw1 = LOAD32_L(xv +  4),
         xw2 = LOAD32_L(xv +  8), xw3 = LOAD32_L(xv + 12),
         xw4 = LOAD32_L(xv + 16), xw5 = LOAD32_L(xv + 20),
         xw6 = LOAD32_L(xv + 24), xw7 = LOAD32_L(xv + 28);
  piece PIECES(x), b, c;

  /* First, split the 32-bit words into the irregularly-sized pieces we need
   * for the field representation.  These pieces are all positive.  We'll do
   * the sign correction afterwards.
   *
   * It may be that the top bit of the input is set.  Avoid trouble by
   * folding that back round into the bottom piece of the representation.
   *
   * Here, we briefly have 0 <= x_0 < 2^26 + 19, but will resolve this later.
   * Otherwise, we have 0 <= x_{2i} < 2^26, and 0 <= x_{2i+1} < 2^25.
   */
  x0 = ((xw0 <<  0)&0x03ffffff) + 19*((xw7 >> 31)&0x00000001);
  x1 = ((xw1 <<  6)&0x01ffffc0) |    ((xw0 >> 26)&0x0000003f);
  x2 = ((xw2 << 13)&0x03ffe000) |    ((xw1 >> 19)&0x00001fff);
  x3 = ((xw3 << 19)&0x01f80000) |    ((xw2 >> 13)&0x0007ffff);
  x4 =                               ((xw3 >>  6)&0x03ffffff);
  x5 =  (xw4 <<  0)&0x01ffffff;
  x6 = ((xw5 <<  7)&0x03ffff80) |    ((xw4 >> 25)&0x0000007f);
  x7 = ((xw6 << 13)&0x01ffe000) |    ((xw5 >> 19)&0x00001fff);
  x8 = ((xw7 << 20)&0x03f00000) |    ((xw6 >> 12)&0x000fffff);
  x9 =                               ((xw7 >>  6)&0x01ffffff);

  /* Next, we convert these pieces into a roughly balanced signed
   * representation.  For each piece, check to see if its top bit is set.  If
   * it is, then lend a bit to the next piece over.  For x_9, this needs to
   * be carried around, which is a little fiddly.  In particular, we delay
   * the carry until after all of the pieces have been balanced.  If we don't
   * do this, then we have to do a more expensive test for nonzeroness to
   * decide whether to lend a bit leftwards rather than just testing a single
   * bit.
   *
   * This fixes up the anomalous size of x_0: the loan of a bit becomes an
   * actual carry if x_0 >= 2^26.  By the end, then, we have:
   *
   *             { 2^25         if i even
   *    |x_i| <= {
   *             { 2^24         if i odd
   *
   * Note that we don't try for a canonical representation here: both upper
   * and lower bounds are achievable.
   *
   * All of the x_i at this point are positive, so we don't need to do
   * anything wierd when masking them.
   */
  b = x9&B24; c   = 19&((b >> 19) - (b >> 24)); x9 -= b << 1;
  b = x8&B25; x9 +=      b >> 25;               x8 -= b << 1;
  b = x7&B24; x8 +=      b >> 24;               x7 -= b << 1;
  b = x6&B25; x7 +=      b >> 25;               x6 -= b << 1;
  b = x5&B24; x6 +=      b >> 24;               x5 -= b << 1;
  b = x4&B25; x5 +=      b >> 25;               x4 -= b << 1;
  b = x3&B24; x4 +=      b >> 24;               x3 -= b << 1;
  b = x2&B25; x3 +=      b >> 25;               x2 -= b << 1;
  b = x1&B24; x2 +=      b >> 24;               x1 -= b << 1;
  b = x0&B25; x1 +=     (b >> 25) + (x0 >> 26); x0 = (x0&M26) - (b << 1);
              x0 +=      c;

  /* And with that, we're done. */
  STASH(z, x);

#elif F25519_IMPL == 10

  piece x[NPIECE];
  unsigned i, j, n, wd;
  uint32 a;
  int b, c;

  /* First, just get the content out of the buffer. */
  for (i = j = a = n = 0, wd = 10; j < NPIECE; i++) {
    a |= (uint32)xv[i] << n; n += 8;
    if (n >= wd) {
      x[j++] = a&MASK(wd);
      a >>= wd; n -= wd;
      wd = PIECEWD(j);
    }
  }

  /* There's a little bit left over from the top byte.  Carry it into the low
   * piece.
   */
  x[0] += 19*(int)(a&MASK(n));

  /* Next, convert the pieces into a roughly balanced signed representation.
   * If a piece's top bit is set, lend a bit to the next piece over.  For
   * x_25, this needs to be carried around, which is a bit fiddly.
   */
  b = x[NPIECE - 1]&B8;
  c = 19&((b >> 3) - (b >> 8));
  x[NPIECE - 1] -= b << 1;
  for (i = NPIECE - 2; i > 0; i--) {
    wd = PIECEWD(i) - 1;
    b = x[i]&BIT(wd);
    x[i + 1] += b >> wd;
    x[i] -= b << 1;
  }
  b = x[0]&B9;
  x[1] += (b >> 9) + (x[0] >> 10);
  x[0] = (x[0]&M10) - (b << 1) + c;

  /* And we're done. */
  for (i = 0; i < NPIECE; i++) z->P[i] = x[i];

#endif
}

/* --- @f25519_store@ --- *
 *
 * Arguments:   @octet zv[32]@ = where to write the result
 *              @const f25519 *x@ = the field element to write
 *
 * Returns:     ---
 *
 * Use:         Stores a field element in the given octet vector in external
 *              representation.  A canonical encoding is always stored, so,
 *              in particular, the top bit of @xv[31]@ is always left clear.
 */

void f25519_store(octet zv[32], const f25519 *x)
{
#if F25519_IMPL == 26

  piece PIECES(x), PIECES(y), c, d;
  uint32 zw0, zw1, zw2, zw3, zw4, zw5, zw6, zw7;
  mask32 m;

  FETCH(x, x);

  /* First, propagate the carries throughout the pieces.  By the end of this,
   * we'll have all of the pieces canonically sized and positive, and maybe
   * there'll be (signed) carry out.  The carry c is in { -1, 0, +1 }, and
   * the remaining value will be in the half-open interval [0, 2^255).  The
   * whole represented value is then x + 2^255 c.
   *
   * It's worth paying careful attention to the bounds.  We assume that we
   * start out with |x_i| <= 2^30.  We start by cutting off and reducing the
   * carry c_9 from the topmost piece, x_9.  This leaves 0 <= x_9 < 2^25; and
   * we'll have |c_9| <= 2^5.  We multiply this by 19 and we'll add it onto
   * x_0 and propagate the carries: but what bounds can we calculate on x
   * before this?
   *
   * Let o_i = floor(51 i/2).  We have X_i = SUM_{0<=j<i} x_j 2^{o_i}, so
   * x = X_10.  We see, inductively, that |X_i| < 2^{31+o_{i-1}}: X_0 = 0;
   * |x_i| <= 2^30; and |X_{i+1}| = |X_i + x_i 2^{o_i}| <= |X_i| + 2^{30+o_i}
   * < 2^{31+o_i}.  Then x = X_9 + 2^230 x_9, and we have better bounds for
   * x_9, so
   *
   *    -2^235 < x + 19 c_9 < 2^255 + 2^235
   *
   * Here, the x_i are signed, so we must be cautious about bithacking them.
   */
              c = ASR(piece, x9, 25); x9 = (upiece)x9&M25;
  x0 += 19*c; c = ASR(piece, x0, 26); x0 = (upiece)x0&M26;
  x1 +=    c; c = ASR(piece, x1, 25); x1 = (upiece)x1&M25;
  x2 +=    c; c = ASR(piece, x2, 26); x2 = (upiece)x2&M26;
  x3 +=    c; c = ASR(piece, x3, 25); x3 = (upiece)x3&M25;
  x4 +=    c; c = ASR(piece, x4, 26); x4 = (upiece)x4&M26;
  x5 +=    c; c = ASR(piece, x5, 25); x5 = (upiece)x5&M25;
  x6 +=    c; c = ASR(piece, x6, 26); x6 = (upiece)x6&M26;
  x7 +=    c; c = ASR(piece, x7, 25); x7 = (upiece)x7&M25;
  x8 +=    c; c = ASR(piece, x8, 26); x8 = (upiece)x8&M26;
  x9 +=    c; c = ASR(piece, x9, 25); x9 = (upiece)x9&M25;

  /* Now we have a slightly fiddly job to do.  If c = +1, or if c = 0 and
   * x >= 2^255 - 19, then we should subtract 2^255 - 19 from the whole
   * value; if c = -1 then we should add 2^255 - 19; and otherwise we should
   * do nothing.
   *
   * But conditional behaviour is bad, m'kay.  So here's what we do instead.
   *
   * The first job is to sort out what we wanted to do.  If c = -1 then we
   * want to (a) invert the constant addend and (b) feed in a carry-in;
   * otherwise, we don't.
   */
  m = SIGN(c);
  d = m&1;

  /* Now do the addition/subtraction.  Remember that all of the x_i are
   * nonnegative, so shifting and masking are safe and easy.
   */
  d += x0 + (19 ^ (M26&m)); y0 = d&M26; d >>= 26;
  d += x1 +       (M25&m);  y1 = d&M25; d >>= 25;
  d += x2 +       (M26&m);  y2 = d&M26; d >>= 26;
  d += x3 +       (M25&m);  y3 = d&M25; d >>= 25;
  d += x4 +       (M26&m);  y4 = d&M26; d >>= 26;
  d += x5 +       (M25&m);  y5 = d&M25; d >>= 25;
  d += x6 +       (M26&m);  y6 = d&M26; d >>= 26;
  d += x7 +       (M25&m);  y7 = d&M25; d >>= 25;
  d += x8 +       (M26&m);  y8 = d&M26; d >>= 26;
  d += x9 +       (M25&m);  y9 = d&M25; d >>= 25;

  /* The final carry-out is in d; since we only did addition, and the x_i are
   * nonnegative, then d is in { 0, 1 }.  We want to keep y, rather than x,
   * if (a) c /= 0 (in which case we know that the old value was
   * unsatisfactory), or (b) if d = 1 (in which case, if c = 0, we know that
   * the subtraction didn't cause a borrow, so we must be in the case where
   * 2^255 - 19 <= x < 2^255).
   */
  m = NONZEROP(c) | ~NONZEROP(d - 1);
  x0 = (y0&m) | (x0&~m); x1 = (y1&m) | (x1&~m);
  x2 = (y2&m) | (x2&~m); x3 = (y3&m) | (x3&~m);
  x4 = (y4&m) | (x4&~m); x5 = (y5&m) | (x5&~m);
  x6 = (y6&m) | (x6&~m); x7 = (y7&m) | (x7&~m);
  x8 = (y8&m) | (x8&~m); x9 = (y9&m) | (x9&~m);

  /* Extract 32-bit words from the value. */
  zw0 = ((x0 >>  0)&0x03ffffff) | (((uint32)x1 << 26)&0xfc000000);
  zw1 = ((x1 >>  6)&0x0007ffff) | (((uint32)x2 << 19)&0xfff80000);
  zw2 = ((x2 >> 13)&0x00001fff) | (((uint32)x3 << 13)&0xffffe000);
  zw3 = ((x3 >> 19)&0x0000003f) | (((uint32)x4 <<  6)&0xffffffc0);
  zw4 = ((x5 >>  0)&0x01ffffff) | (((uint32)x6 << 25)&0xfe000000);
  zw5 = ((x6 >>  7)&0x0007ffff) | (((uint32)x7 << 19)&0xfff80000);
  zw6 = ((x7 >> 13)&0x00000fff) | (((uint32)x8 << 12)&0xfffff000);
  zw7 = ((x8 >> 20)&0x0000003f) | (((uint32)x9 <<  6)&0x7fffffc0);

  /* Store the result as an octet string. */
  STORE32_L(zv +  0, zw0); STORE32_L(zv +  4, zw1);
  STORE32_L(zv +  8, zw2); STORE32_L(zv + 12, zw3);
  STORE32_L(zv + 16, zw4); STORE32_L(zv + 20, zw5);
  STORE32_L(zv + 24, zw6); STORE32_L(zv + 28, zw7);

#elif F25519_IMPL == 10

  piece y[NPIECE], yy[NPIECE], c, d;
  unsigned i, j, n, wd;
  uint32 m, a;

  /* Before we do anything, copy the input so we can hack on it. */
  for (i = 0; i < NPIECE; i++) y[i] = x->P[i];

  /* First, propagate the carries throughout the pieces.
   *
   * It's worth paying careful attention to the bounds.  We assume that we
   * start out with |y_i| <= 2^14.  We start by cutting off and reducing the
   * carry c_25 from the topmost piece, y_25.  This leaves 0 <= y_25 < 2^9;
   * and we'll have |c_25| <= 2^5.  We multiply this by 19 and we'll ad it
   * onto y_0 and propagte the carries: but what bounds can we calculate on
   * y before this?
   *
   * Let o_i = floor(255 i/26).  We have Y_i = SUM_{0<=j<i} y_j 2^{o_i}, so
   * y = Y_26.  We see, inductively, that |Y_i| < 2^{31+o_{i-1}}: Y_0 = 0;
   * |y_i| <= 2^14; and |Y_{i+1}| = |Y_i + y_i 2^{o_i}| <= |Y_i| + 2^{14+o_i}
   * < 2^{15+o_i}.  Then x = Y_25 + 2^246 y_25, and we have better bounds for
   * y_25, so
   *
   *    -2^251 < y + 19 c_25 < 2^255 + 2^251
   *
   * Here, the y_i are signed, so we must be cautious about bithacking them.
   *
   * (Rather closer than the 10-piece case above, but still doable in one
   * pass.)
   */
  c = 19*ASR(piece, y[NPIECE - 1], 9);
  y[NPIECE - 1] = (upiece)y[NPIECE - 1]&M9;
  for (i = 0; i < NPIECE; i++) {
    wd = PIECEWD(i);
    y[i] += c;
    c = ASR(piece, y[i], wd);
    y[i] = (upiece)y[i]&MASK(wd);
  }

  /* Now the addition or subtraction. */
  m = SIGN(c);
  d = m&1;

  d += y[0] + (19 ^ (M10&m));
  yy[0] = d&M10;
  d >>= 10;
  for (i = 1; i < NPIECE; i++) {
    wd = PIECEWD(i);
    d += y[i] + (MASK(wd)&m);
    yy[i] = d&MASK(wd);
    d >>= wd;
  }

  /* Choose which value to keep. */
  m = NONZEROP(c) | ~NONZEROP(d - 1);
  for (i = 0; i < NPIECE; i++) y[i] = (yy[i]&m) | (y[i]&~m);

  /* Store the result as an octet string. */
  for (i = j = a = n = 0; i < NPIECE; i++) {
    a |= (upiece)y[i] << n; n += PIECEWD(i);
    while (n >= 8) {
      zv[j++] = a&0xff;
      a >>= 8; n -= 8;
    }
  }
  zv[j++] = a;

#endif
}

/* --- @f25519_set@ --- *
 *
 * Arguments:   @f25519 *z@ = where to write the result
 *              @int a@ = a small-ish constant
 *
 * Returns:     ---
 *
 * Use:         Sets @z@ to equal @a@.
 */

void f25519_set(f25519 *x, int a)
{
  unsigned i;

  x->P[0] = a;
  for (i = 1; i < NPIECE; i++) x->P[i] = 0;
}

/*----- Basic arithmetic --------------------------------------------------*/

/* --- @f25519_add@ --- *
 *
 * Arguments:   @f25519 *z@ = where to put the result (may alias @x@ or @y@)
 *              @const f25519 *x, *y@ = two operands
 *
 * Returns:     ---
 *
 * Use:         Set @z@ to the sum %$x + y$%.
 */

void f25519_add(f25519 *z, const f25519 *x, const f25519 *y)
{
#if F25519_IMPL == 26
  z->P[0] = x->P[0] + y->P[0]; z->P[1] = x->P[1] + y->P[1];
  z->P[2] = x->P[2] + y->P[2]; z->P[3] = x->P[3] + y->P[3];
  z->P[4] = x->P[4] + y->P[4]; z->P[5] = x->P[5] + y->P[5];
  z->P[6] = x->P[6] + y->P[6]; z->P[7] = x->P[7] + y->P[7];
  z->P[8] = x->P[8] + y->P[8]; z->P[9] = x->P[9] + y->P[9];
#elif F25519_IMPL == 10
  unsigned i;
  for (i = 0; i < NPIECE; i++) z->P[i] = x->P[i] + y->P[i];
#endif
}

/* --- @f25519_sub@ --- *
 *
 * Arguments:   @f25519 *z@ = where to put the result (may alias @x@ or @y@)
 *              @const f25519 *x, *y@ = two operands
 *
 * Returns:     ---
 *
 * Use:         Set @z@ to the difference %$x - y$%.
 */

void f25519_sub(f25519 *z, const f25519 *x, const f25519 *y)
{
#if F25519_IMPL == 26
  z->P[0] = x->P[0] - y->P[0]; z->P[1] = x->P[1] - y->P[1];
  z->P[2] = x->P[2] - y->P[2]; z->P[3] = x->P[3] - y->P[3];
  z->P[4] = x->P[4] - y->P[4]; z->P[5] = x->P[5] - y->P[5];
  z->P[6] = x->P[6] - y->P[6]; z->P[7] = x->P[7] - y->P[7];
  z->P[8] = x->P[8] - y->P[8]; z->P[9] = x->P[9] - y->P[9];
#elif F25519_IMPL == 10
  unsigned i;
  for (i = 0; i < NPIECE; i++) z->P[i] = x->P[i] - y->P[i];
#endif
}

/*----- Constant-time utilities -------------------------------------------*/

/* --- @f25519_condswap@ --- *
 *
 * Arguments:   @f25519 *x, *y@ = two operands
 *              @uint32 m@ = a mask
 *
 * Returns:     ---
 *
 * Use:         If @m@ is zero, do nothing; if @m@ is all-bits-set, then
 *              exchange @x@ and @y@.  If @m@ has some other value, then
 *              scramble @x@ and @y@ in an unhelpful way.
 */

void f25519_condswap(f25519 *x, f25519 *y, uint32 m)
{
  mask32 mm = FIX_MASK32(m);

#if F25519_IMPL == 26
  CONDSWAP(x->P[0], y->P[0], mm);
  CONDSWAP(x->P[1], y->P[1], mm);
  CONDSWAP(x->P[2], y->P[2], mm);
  CONDSWAP(x->P[3], y->P[3], mm);
  CONDSWAP(x->P[4], y->P[4], mm);
  CONDSWAP(x->P[5], y->P[5], mm);
  CONDSWAP(x->P[6], y->P[6], mm);
  CONDSWAP(x->P[7], y->P[7], mm);
  CONDSWAP(x->P[8], y->P[8], mm);
  CONDSWAP(x->P[9], y->P[9], mm);
#elif F25519_IMPL == 10
  unsigned i;
  for (i = 0; i < NPIECE; i++) CONDSWAP(x->P[i], y->P[i], mm);
#endif
}

/*----- Multiplication ----------------------------------------------------*/

#if F25519_IMPL == 26

/* Let B = 2^63 - 1 be the largest value such that +B and -B can be
 * represented in a double-precision piece.  On entry, it must be the case
 * that |X_i| <= M <= B - 2^25 for some M.  If this is the case, then, on
 * exit, we will have |Z_i| <= 2^25 + 19 M/2^25.
 */
#define CARRYSTEP(z, x, m, b, f, xx, n) do {                            \
  (z) = (dblpiece)((udblpiece)(x)&(m)) - (b) +                          \
    (f)*ASR(dblpiece, (xx), (n));                                       \
} while (0)
#define CARRY_REDUCE(z, x) do {                                         \
  dblpiece PIECES(_t);                                                  \
                                                                        \
  /* Bias the input pieces.  This keeps the carries and so on centred   \
   * around zero rather than biased positive.                           \
   */                                                                   \
  _t0 = (x##0) + B25; _t1 = (x##1) + B24;                               \
  _t2 = (x##2) + B25; _t3 = (x##3) + B24;                               \
  _t4 = (x##4) + B25; _t5 = (x##5) + B24;                               \
  _t6 = (x##6) + B25; _t7 = (x##7) + B24;                               \
  _t8 = (x##8) + B25; _t9 = (x##9) + B24;                               \
                                                                        \
  /* Calculate the reduced pieces.  Careful with the bithacking. */     \
  CARRYSTEP(z##0, _t0, M26, B25, 19, _t9, 25);                          \
  CARRYSTEP(z##1, _t1, M25, B24,  1, _t0, 26);                          \
  CARRYSTEP(z##2, _t2, M26, B25,  1, _t1, 25);                          \
  CARRYSTEP(z##3, _t3, M25, B24,  1, _t2, 26);                          \
  CARRYSTEP(z##4, _t4, M26, B25,  1, _t3, 25);                          \
  CARRYSTEP(z##5, _t5, M25, B24,  1, _t4, 26);                          \
  CARRYSTEP(z##6, _t6, M26, B25,  1, _t5, 25);                          \
  CARRYSTEP(z##7, _t7, M25, B24,  1, _t6, 26);                          \
  CARRYSTEP(z##8, _t8, M26, B25,  1, _t7, 25);                          \
  CARRYSTEP(z##9, _t9, M25, B24,  1, _t8, 26);                          \
} while (0)

#elif F25519_IMPL == 10

/* Perform carry propagation on X. */
static void carry_reduce(dblpiece x[NPIECE])
{
  /* Initial bounds: we assume |x_i| < 2^31 - 2^27. */

  unsigned i, j;
  dblpiece c;

  /* The result is nearly canonical, because we do sequential carry
   * propagation, because smaller processors are more likely to prefer the
   * smaller working set than the instruction-level parallelism.
   *
   * Start at x_23; truncate it to 10 bits, and propagate the carry to x_24.
   * Truncate x_24 to 10 bits, and add the carry onto x_25.  Truncate x_25 to
   * 9 bits, and add 19 times the carry onto x_0.  And so on.
   *
   * Let c_i be the portion of x_i to be carried onto x_{i+1}.  I claim that
   * |c_i| <= 2^22.  Then the carry /into/ any x_i has magnitude at most
   * 19*2^22 < 2^27 (allowing for the reduction as we carry from x_25 to
   * x_0), and x_i after carry is bounded above by 2^31.  Hence, the carry
   * out is at most 2^22, as claimed.
   *
   * Once we reach x_23 for the second time, we start with |x_23| <= 2^9.
   * The carry into x_23 is at most 2^27 as calculated above; so the carry
   * out into x_24 has magnitude at most 2^17.  In turn, |x_24| <= 2^9 before
   * the carry, so is now no more than 2^18 in magnitude, and the carry out
   * into x_25 is at most 2^8.  This leaves |x_25| < 2^9 after carry
   * propagation.
   *
   * Be careful with the bit hacking because the quantities involved are
   * signed.
   */

  /*For each piece, we bias it so that floor division (as done by an
   * arithmetic right shift) and modulus (as done by bitwise-AND) does the
   * right thing.
   */
#define CARRY(i, wd, b, m) do {                                         \
  x[i] += (b);                                                          \
  c = ASR(dblpiece, x[i], (wd));                                        \
  x[i] = (dblpiece)((udblpiece)x[i]&(m)) - (b);                         \
} while (0)

                             {                CARRY(23, 10, B9, M10);      }
                             { x[24] +=    c; CARRY(24, 10, B9, M10);      }
                             { x[25] +=    c; CARRY(25,  9, B8,  M9);      }
                             {  x[0] += 19*c; CARRY( 0, 10, B9, M10);      }
  for (i = 1; i < 21; ) {
    for (j = i + 4; i < j; ) {  x[i] +=    c; CARRY( i, 10, B9, M10); i++; }
                             {  x[i] +=    c; CARRY( i,  9, B8,  M9); i++; }
  }
  while (i < 25)             {  x[i] +=    c; CARRY( i, 10, B9, M10); i++; }
  x[25] += c;

#undef CARRY
}

#endif

/* --- @f25519_mulconst@ --- *
 *
 * Arguments:   @f25519 *z@ = where to put the result (may alias @x@)
 *              @const f25519 *x@ = an operand
 *              @long a@ = a small-ish constant; %$|a| < 2^{20}$%.
 *
 * Returns:     ---
 *
 * Use:         Set @z@ to the product %$a x$%.
 */

void f25519_mulconst(f25519 *z, const f25519 *x, long a)
{
#if F25519_IMPL == 26

  piece PIECES(x);
  dblpiece PIECES(z), aa = a;

  FETCH(x, x);

  /* Suppose that |x_i| <= 2^27, and |a| <= 2^23.  Then we'll have
   * |z_i| <= 2^50.
   */
  z0 = aa*x0; z1 = aa*x1; z2 = aa*x2; z3 = aa*x3; z4 = aa*x4;
  z5 = aa*x5; z6 = aa*x6; z7 = aa*x7; z8 = aa*x8; z9 = aa*x9;

  /* Following `CARRY_REDUCE', we'll have |z_i| <= 2^26. */
  CARRY_REDUCE(z, z);
  STASH(z, z);

#elif F25519_IMPL == 10

  dblpiece y[NPIECE];
  unsigned i;

  for (i = 0; i < NPIECE; i++) y[i] = a*x->P[i];
  carry_reduce(y);
  for (i = 0; i < NPIECE; i++) z->P[i] = y[i];

#endif
}

/* --- @f25519_mul@ --- *
 *
 * Arguments:   @f25519 *z@ = where to put the result (may alias @x@ or @y@)
 *              @const f25519 *x, *y@ = two operands
 *
 * Returns:     ---
 *
 * Use:         Set @z@ to the product %$x y$%.
 */

void f25519_mul(f25519 *z, const f25519 *x, const f25519 *y)
{
#if F25519_IMPL == 26

  piece PIECES(x), PIECES(y);
  dblpiece PIECES(z);
  unsigned i;

  FETCH(x, x); FETCH(y, y);

  /* Suppose that |x_i|, |y_i| <= 2^27.  Then we'll have
   *
   *    |z_0| <= 267*2^54
   *    |z_1| <= 154*2^54
   *    |z_2| <= 213*2^54
   *    |z_3| <= 118*2^54
   *    |z_4| <= 159*2^54
   *    |z_5| <=  82*2^54
   *    |z_6| <= 105*2^54
   *    |z_7| <=  46*2^54
   *    |z_8| <=  51*2^54
   *    |z_9| <=  10*2^54
   *
   * all of which are less than 2^63 - 2^25.
   */

#define M(a, b) ((dblpiece)(a)*(b))
  z0 =     M(x0, y0) +
       19*(M(x2, y8) + M(x4, y6) + M(x6, y4) + M(x8, y2)) +
       38*(M(x1, y9) + M(x3, y7) + M(x5, y5) + M(x7, y3) + M(x9, y1));
  z1 =     M(x0, y1) + M(x1, y0) +
       19*(M(x2, y9) + M(x3, y8) + M(x4, y7) + M(x5, y6) +
           M(x6, y5) + M(x7, y4) + M(x8, y3) + M(x9, y2));
  z2 =     M(x0, y2) + M(x2, y0) +
        2* M(x1, y1) +
       19*(M(x4, y8) + M(x6, y6) + M(x8, y4)) +
       38*(M(x3, y9) + M(x5, y7) + M(x7, y5) + M(x9, y3));
  z3 =     M(x0, y3) + M(x1, y2) + M(x2, y1) + M(x3, y0) +
       19*(M(x4, y9) + M(x5, y8) + M(x6, y7) +
           M(x7, y6) + M(x8, y5) + M(x9, y4));
  z4 =     M(x0, y4) + M(x2, y2) + M(x4, y0) +
        2*(M(x1, y3) + M(x3, y1)) +
       19*(M(x6, y8) + M(x8, y6)) +
       38*(M(x5, y9) + M(x7, y7) + M(x9, y5));
  z5 =     M(x0, y5) + M(x1, y4) + M(x2, y3) +
           M(x3, y2) + M(x4, y1) + M(x5, y0) +
       19*(M(x6, y9) + M(x7, y8) + M(x8, y7) + M(x9, y6));
  z6 =     M(x0, y6) + M(x2, y4) + M(x4, y2) + M(x6, y0) +
        2*(M(x1, y5) + M(x3, y3) + M(x5, y1)) +
       19* M(x8, y8) +
       38*(M(x7, y9) + M(x9, y7));
  z7 =     M(x0, y7) + M(x1, y6) + M(x2, y5) + M(x3, y4) +
           M(x4, y3) + M(x5, y2) + M(x6, y1) + M(x7, y0) +
       19*(M(x8, y9) + M(x9, y8));
  z8 =     M(x0, y8) + M(x2, y6) + M(x4, y4) + M(x6, y2) + M(x8, y0) +
        2*(M(x1, y7) + M(x3, y5) + M(x5, y3) + M(x7, y1)) +
       38* M(x9, y9);
  z9 =     M(x0, y9) + M(x1, y8) + M(x2, y7) + M(x3, y6) + M(x4, y5) +
           M(x5, y4) + M(x6, y3) + M(x7, y2) + M(x8, y1) + M(x9, y0);
#undef M

  /* From above, we have |z_i| <= 2^63 - 2^25.  A pass of `CARRY_REDUCE' will
   * leave |z_i| <= 2^38 + 2^25; and a second pass will leave |z_i| <= 2^25 +
   * 2^13, which is comfortable for an addition prior to the next
   * multiplication.
   */
  for (i = 0; i < 2; i++) CARRY_REDUCE(z, z);
  STASH(z, z);

#elif F25519_IMPL == 10

  dblpiece u[NPIECE], t, tt, p;
  unsigned i, j, k;

  /* This is unpleasant.  Honestly, this table seems to be the best way of
   * doing it.
   */
  static const unsigned short off[NPIECE] = {
      0,  10,  20,  30,  40,  50,  59,  69,  79,  89,  99, 108, 118,
    128, 138, 148, 157, 167, 177, 187, 197, 206, 216, 226, 236, 246
  };

  /* First pass: things we must multiply by 19 or 38. */
  for (i = 0; i < NPIECE - 1; i++) {
    t = tt = 0;
    for (j = i + 1; j < NPIECE; j++) {
      k = NPIECE + i - j; p = (dblpiece)x->P[j]*y->P[k];
      if (off[i] < off[j] + off[k] - 255) tt += p;
      else t += p;
    }
    u[i] = 19*(t + 2*tt);
  }
  u[NPIECE - 1] = 0;

  /* Second pass: things we must multiply by 1 or 2. */
  for (i = 0; i < NPIECE; i++) {
    t = tt = 0;
    for (j = 0; j <= i; j++) {
      k = i - j; p = (dblpiece)x->P[j]*y->P[k];
      if (off[i] < off[j] + off[k]) tt += p;
      else t += p;
    }
    u[i] += t + 2*tt;
  }

  /* And we're done. */
  carry_reduce(u);
  for (i = 0; i < NPIECE; i++) z->P[i] = u[i];

#endif
}

/* --- @f25519_sqr@ --- *
 *
 * Arguments:   @f25519 *z@ = where to put the result (may alias @x@ or @y@)
 *              @const f25519 *x@ = an operand
 *
 * Returns:     ---
 *
 * Use:         Set @z@ to the square %$x^2$%.
 */

void f25519_sqr(f25519 *z, const f25519 *x)
{
#if F25519_IMPL == 26

  piece PIECES(x);
  dblpiece PIECES(z);
  unsigned i;

  FETCH(x, x);

  /* See `f25519_mul' for bounds. */

#define M(a, b) ((dblpiece)(a)*(b))
  z0 =     M(x0, x0) +
       38*(M(x2, x8) + M(x4, x6) + M(x5, x5)) +
       76*(M(x1, x9) + M(x3, x7));
  z1 =  2* M(x0, x1) +
       38*(M(x2, x9) + M(x3, x8) + M(x4, x7) + M(x5, x6));
  z2 =  2*(M(x0, x2) + M(x1, x1)) +
       19* M(x6, x6) +
       38* M(x4, x8) +
       76*(M(x3, x9) + M(x5, x7));
  z3 =  2*(M(x0, x3) + M(x1, x2)) +
       38*(M(x4, x9) + M(x5, x8) + M(x6, x7));
  z4 =     M(x2, x2) +
        2* M(x0, x4) +
        4* M(x1, x3) +
       38*(M(x6, x8) + M(x7, x7)) +
       76* M(x5, x9);
  z5 =  2*(M(x0, x5) + M(x1, x4) + M(x2, x3)) +
       38*(M(x6, x9) + M(x7, x8));
  z6 =  2*(M(x0, x6) + M(x2, x4) + M(x3, x3)) +
        4* M(x1, x5) +
       19* M(x8, x8) +
       76* M(x7, x9);
  z7 =  2*(M(x0, x7) + M(x1, x6) + M(x2, x5) + M(x3, x4)) +
       38* M(x8, x9);
  z8 =     M(x4, x4) +
        2*(M(x0, x8) + M(x2, x6)) +
        4*(M(x1, x7) + M(x3, x5)) +
       38* M(x9, x9);
  z9 =  2*(M(x0, x9) + M(x1, x8) + M(x2, x7) + M(x3, x6) + M(x4, x5));
#undef M

  /* See `f25519_mul' for details. */
  for (i = 0; i < 2; i++) CARRY_REDUCE(z, z);
  STASH(z, z);

#elif F25519_IMPL == 10
  f25519_mul(z, x, x);
#endif
}

/*----- More complicated things -------------------------------------------*/

/* --- @f25519_inv@ --- *
 *
 * Arguments:   @f25519 *z@ = where to put the result (may alias @x@)
 *              @const f25519 *x@ = an operand
 *
 * Returns:     ---
 *
 * Use:         Stores in @z@ the multiplicative inverse %$x^{-1}$%.  If
 *              %$x = 0$% then @z@ is set to zero.  This is considered a
 *              feature.
 */

void f25519_inv(f25519 *z, const f25519 *x)
{
  f25519 t, u, t2, t11, t2p10m1, t2p50m1;
  unsigned i;

#define SQRN(z, x, n) do {                                              \
  f25519_sqr((z), (x));                                                 \
  for (i = 1; i < (n); i++) f25519_sqr((z), (z));                       \
} while (0)

  /* Calculate x^-1 = x^(p - 2) = x^(2^255 - 21), which also handles x = 0 as
   * intended.  The addition chain here is from Bernstein's implementation; I
   * couldn't find a better one.
   */                                   /* step | value */
  f25519_sqr(&t2, x);                   /*    1 | 2 */
  SQRN(&u, &t2, 2);                     /*    3 | 8 */
  f25519_mul(&t, &u, x);                /*    4 | 9 */
  f25519_mul(&t11, &t, &t2);            /*    5 | 11 = 2^5 - 21 */
  f25519_sqr(&u, &t11);                 /*    6 | 22 */
  f25519_mul(&t, &t, &u);               /*    7 | 31 = 2^5 - 1 */
  SQRN(&u, &t, 5);                      /*   12 | 2^10 - 2^5 */
  f25519_mul(&t2p10m1, &t, &u);         /*   13 | 2^10 - 1 */
  SQRN(&u, &t2p10m1, 10);               /*   23 | 2^20 - 2^10 */
  f25519_mul(&t, &t2p10m1, &u);         /*   24 | 2^20 - 1 */
  SQRN(&u, &t, 20);                     /*   44 | 2^40 - 2^20 */
  f25519_mul(&t, &t, &u);               /*   45 | 2^40 - 1 */
  SQRN(&u, &t, 10);                     /*   55 | 2^50 - 2^10 */
  f25519_mul(&t2p50m1, &t2p10m1, &u);   /*   56 | 2^50 - 1 */
  SQRN(&u, &t2p50m1, 50);               /*  106 | 2^100 - 2^50 */
  f25519_mul(&t, &t2p50m1, &u);         /*  107 | 2^100 - 1 */
  SQRN(&u, &t, 100);                    /*  207 | 2^200 - 2^100 */
  f25519_mul(&t, &t, &u);               /*  208 | 2^200 - 1 */
  SQRN(&u, &t, 50);                     /*  258 | 2^250 - 2^50 */
  f25519_mul(&t, &t2p50m1, &u);         /*  259 | 2^250 - 1 */
  SQRN(&u, &t, 5);                      /*  264 | 2^255 - 2^5 */
  f25519_mul(z, &u, &t11);              /*  265 | 2^255 - 21 */

#undef SQRN
}

/*----- Test rig ----------------------------------------------------------*/

#ifdef TEST_RIG

#include <mLib/report.h>
#include <mLib/testrig.h>

static void fixdstr(dstr *d)
{
  if (d->len > 32)
    die(1, "invalid length for f25519");
  else if (d->len < 32) {
    dstr_ensure(d, 32);
    memset(d->buf + d->len, 0, 32 - d->len);
    d->len = 32;
  }
}

static void cvt_f25519(const char *buf, dstr *d)
{
  dstr dd = DSTR_INIT;

  type_hex.cvt(buf, &dd); fixdstr(&dd);
  dstr_ensure(d, sizeof(f25519)); d->len = sizeof(f25519);
  f25519_load((f25519 *)d->buf, (const octet *)dd.buf);
  dstr_destroy(&dd);
}

static void dump_f25519(dstr *d, FILE *fp)
  { fdump(stderr, "???", (const piece *)d->buf); }

static void cvt_f25519_ref(const char *buf, dstr *d)
  { type_hex.cvt(buf, d); fixdstr(d); }

static void dump_f25519_ref(dstr *d, FILE *fp)
{
  f25519 x;

  f25519_load(&x, (const octet *)d->buf);
  fdump(stderr, "???", x.P);
}

static int eq(const f25519 *x, dstr *d)
  { octet b[32]; f25519_store(b, x); return (memcmp(b, d->buf, 32) == 0); }

static const test_type
  type_f25519 = { cvt_f25519, dump_f25519 },
  type_f25519_ref = { cvt_f25519_ref, dump_f25519_ref };

#define TEST_UNOP(op)                                                   \
  static int vrf_##op(dstr dv[])                                        \
  {                                                                     \
    f25519 *x = (f25519 *)dv[0].buf;                                    \
    f25519 z, zz;                                                       \
    int ok = 1;                                                         \
                                                                        \
    f25519_##op(&z, x);                                                 \
    if (!eq(&z, &dv[1])) {                                              \
      ok = 0;                                                           \
      fprintf(stderr, "failed!\n");                                     \
      fdump(stderr, "x", x->P);                                         \
      fdump(stderr, "calc", z.P);                                       \
      f25519_load(&zz, (const octet *)dv[1].buf);                       \
      fdump(stderr, "z", zz.P);                                         \
    }                                                                   \
                                                                        \
    return (ok);                                                        \
  }

TEST_UNOP(sqr)
TEST_UNOP(inv)

#define TEST_BINOP(op)                                                  \
  static int vrf_##op(dstr dv[])                                        \
  {                                                                     \
    f25519 *x = (f25519 *)dv[0].buf, *y = (f25519 *)dv[1].buf;          \
    f25519 z, zz;                                                       \
    int ok = 1;                                                         \
                                                                        \
    f25519_##op(&z, x, y);                                              \
    if (!eq(&z, &dv[2])) {                                              \
      ok = 0;                                                           \
      fprintf(stderr, "failed!\n");                                     \
      fdump(stderr, "x", x->P);                                         \
      fdump(stderr, "y", y->P);                                         \
      fdump(stderr, "calc", z.P);                                       \
      f25519_load(&zz, (const octet *)dv[2].buf);                       \
      fdump(stderr, "z", zz.P);                                         \
    }                                                                   \
                                                                        \
    return (ok);                                                        \
  }

TEST_BINOP(add)
TEST_BINOP(sub)
TEST_BINOP(mul)

static int vrf_mulc(dstr dv[])
{
  f25519 *x = (f25519 *)dv[0].buf;
  long a = *(const long *)dv[1].buf;
  f25519 z, zz;
  int ok = 1;

  f25519_mulconst(&z, x, a);
  if (!eq(&z, &dv[2])) {
    ok = 0;
    fprintf(stderr, "failed!\n");
    fdump(stderr, "x", x->P);
    fprintf(stderr, "a = %ld\n", a);
    fdump(stderr, "calc", z.P);
    f25519_load(&zz, (const octet *)dv[2].buf);
    fdump(stderr, "z", zz.P);
  }

  return (ok);
}

static int vrf_condswap(dstr dv[])
{
  f25519 *x = (f25519 *)dv[0].buf, *y = (f25519 *)dv[1].buf;
  f25519 xx = *x, yy = *y;
  uint32 m = *(uint32 *)dv[2].buf;
  int ok = 1;

  f25519_condswap(&xx, &yy, m);
  if (!eq(&xx, &dv[3]) || !eq(&yy, &dv[4])) {
    ok = 0;
    fprintf(stderr, "failed!\n");
    fdump(stderr, "x", x->P);
    fdump(stderr, "y", y->P);
    fprintf(stderr, "m = 0x%08lx\n", (unsigned long)m);
    fdump(stderr, "calc xx", xx.P);
    fdump(stderr, "calc yy", yy.P);
    f25519_load(&xx, (const octet *)dv[3].buf);
    f25519_load(&yy, (const octet *)dv[4].buf);
    fdump(stderr, "want xx", xx.P);
    fdump(stderr, "want yy", yy.P);
  }

  return (ok);
}

static int vrf_sub_mulc_add_sub_mul(dstr dv[])
{
  f25519 *u = (f25519 *)dv[0].buf, *v = (f25519 *)dv[1].buf,
    *w = (f25519 *)dv[3].buf, *x = (f25519 *)dv[4].buf,
    *y = (f25519 *)dv[5].buf;
  long a = *(const long *)dv[2].buf;
  f25519 umv, aumv, wpaumv, xmy, z, zz;
  int ok = 1;

  f25519_sub(&umv, u, v);
  f25519_mulconst(&aumv, &umv, a);
  f25519_add(&wpaumv, w, &aumv);
  f25519_sub(&xmy, x, y);
  f25519_mul(&z, &wpaumv, &xmy);

  if (!eq(&z, &dv[6])) {
    ok = 0;
    fprintf(stderr, "failed!\n");
    fdump(stderr, "u", u->P);
    fdump(stderr, "v", v->P);
    fdump(stderr, "u - v", umv.P);
    fprintf(stderr, "a = %ld\n", a);
    fdump(stderr, "a (u - v)", aumv.P);
    fdump(stderr, "w + a (u - v)", wpaumv.P);
    fdump(stderr, "x", x->P);
    fdump(stderr, "y", y->P);
    fdump(stderr, "x - y", xmy.P);
    fdump(stderr, "(x - y) (w + a (u - v))", z.P);
    f25519_load(&zz, (const octet *)dv[6].buf); fdump(stderr, "z", zz.P);
  }

  return (ok);
}

static test_chunk tests[] = {
  { "add", vrf_add, { &type_f25519, &type_f25519, &type_f25519_ref } },
  { "sub", vrf_sub, { &type_f25519, &type_f25519, &type_f25519_ref } },
  { "mul", vrf_mul, { &type_f25519, &type_f25519, &type_f25519_ref } },
  { "mulconst", vrf_mulc, { &type_f25519, &type_long, &type_f25519_ref } },
  { "condswap", vrf_condswap,
    { &type_f25519, &type_f25519, &type_uint32,
      &type_f25519_ref, &type_f25519_ref } },
  { "sqr", vrf_sqr, { &type_f25519, &type_f25519_ref } },
  { "inv", vrf_inv, { &type_f25519, &type_f25519_ref } },
  { "sub-mulc-add-sub-mul", vrf_sub_mulc_add_sub_mul,
    { &type_f25519, &type_f25519, &type_long, &type_f25519,
      &type_f25519, &type_f25519, &type_f25519_ref } },
  { 0, 0, { 0 } }
};

int main(int argc, char *argv[])
{
  test_run(argc, argv, tests, SRCDIR "/t/f25519");
  return (0);
}

#endif

/*----- That's all, folks -------------------------------------------------*/