symm/: New SSE2 implementations of Salsa20 and ChaCha.

[catacomb] / symm / salsa20.c

/* -*-c-*-
 *
 * Salsa20 stream cipher
 *
 * (c) 2015 Straylight/Edgeware
 */

/*----- Header files ------------------------------------------------------*/

#include "config.h"

#include <stdarg.h>

#include <mLib/bits.h>

#include "arena.h"
#include "dispatch.h"
#include "gcipher.h"
#include "grand.h"
#include "keysz.h"
#include "paranoia.h"
#include "salsa20.h"
#include "salsa20-core.h"

/*----- Global variables --------------------------------------------------*/

const octet salsa20_keysz[] = { KSZ_SET, 32, 16, 10, 0 };

/*----- The Salsa20 core function and utilities ---------------------------*/

/* --- @core@ --- *
 *
 * Arguments:   @unsigned r@ = number of rounds
 *              @const salsa20_matrix src@ = input matrix
 *              @salsa20_matrix dest@ = where to put the output
 *
 * Returns:     ---
 *
 *
 * Use:         Apply the Salsa20/r core function to @src@, writing the
 *              result to @dest@.  This consists of @r@ rounds followed by
 *              the feedforward step.
 */

CPU_DISPATCH(static, (void),
             void, core, (unsigned r, const salsa20_matrix src,
                          salsa20_matrix dest),
             (r, src, dest),
             pick_core, simple_core);

static void simple_core(unsigned r, const salsa20_matrix src,
                        salsa20_matrix dest)
  { SALSA20_nR(dest, src, r); SALSA20_FFWD(dest, src); }

#ifdef CPUFAM_X86
extern core__functype salsa20_core_x86_sse2;
#endif

static core__functype *pick_core(void)
{
#ifdef CPUFAM_X86
  if (cpu_feature_p(CPUFEAT_X86_SSE2)) return salsa20_core_x86_sse2;
#endif
  return simple_core;
}

/* --- @populate@ --- *
 *
 * Arguments:   @salsa20_matrix a@ = a matrix to fill in
 *              @const void *key@ = pointer to key material
 *              @size_t ksz@ = size of key
 *
 * Returns:     ---
 *
 * Use:         Fills in a Salsa20 matrix from the key, setting the
 *              appropriate constants according to the key length.  The nonce
 *              and position words are left uninitialized.
 */

static void populate(salsa20_matrix a, const void *key, size_t ksz)
{
  const octet *k = key;

  KSZ_ASSERT(salsa20, ksz);

  /* Here's the pattern of key, constant, nonce, and counter pieces in the
   * matrix, before and after our permutation.
   *
   * [ C0 K0 K1 K2 ]       [ C0 C1 C2 C3 ]
   * [ K3 C1 N0 N1 ]  -->  [ K3 T1 K7 K2 ]
   * [ T0 T1 C2 K4 ]       [ T0 K6 K1 N1 ]
   * [ K5 K6 K7 C3 ]       [ K5 K0 N0 K4 ]
   */

  a[13] = LOAD32_L(k +  0);
  a[10] = LOAD32_L(k +  4);
  if (ksz == 10) {
    a[ 7] = LOAD16_L(k +  8);
    a[ 4] = 0;
  } else {
    a[ 7] = LOAD32_L(k +  8);
    a[ 4] = LOAD32_L(k + 12);
  }
  if (ksz <= 16) {
    a[15] = a[13];
    a[12] = a[10];
    a[ 9] = a[ 7];
    a[ 6] = a[ 4];
    a[ 0] = SALSA20_A128;
    a[ 1] = SALSA20_B128;
    a[ 2] = ksz == 10 ? SALSA20_C80 : SALSA20_C128;
    a[ 3] = SALSA20_D128;
  } else {
    a[15] = LOAD32_L(k + 16);
    a[12] = LOAD32_L(k + 20);
    a[ 9] = LOAD32_L(k + 24);
    a[ 6] = LOAD32_L(k + 28);
    a[ 0] = SALSA20_A256;
    a[ 1] = SALSA20_B256;
    a[ 2] = SALSA20_C256;
    a[ 3] = SALSA20_D256;
  }
}

/*----- Salsa20 implementation --------------------------------------------*/

/* --- @salsa20_init@ --- *
 *
 * Arguments:   @salsa20_ctx *ctx@ = context to fill in
 *              @const void *key@ = pointer to key material
 *              @size_t ksz@ = size of key (either 32 or 16)
 *              @const void *nonce@ = initial nonce, or null
 *
 * Returns:     ---
 *
 * Use:         Initializes a Salsa20 context ready for use.
 */

void salsa20_init(salsa20_ctx *ctx, const void *key, size_t ksz,
                  const void *nonce)
{
  static const octet zerononce[SALSA20_NONCESZ];

  populate(ctx->a, key, ksz);
  salsa20_setnonce(ctx, nonce ? nonce : zerononce);
}

/* --- @salsa20_setnonce@ --- *
 *
 * Arguments:   @salsa20_ctx *ctx@ = pointer to context
 *              @const void *nonce@ = the nonce (@SALSA20_NONCESZ@ bytes)
 *
 * Returns:     ---
 *
 * Use:         Set a new nonce in the context @ctx@, e.g., for processing a
 *              different message.  The stream position is reset to zero (see
 *              @salsa20_seek@ etc.).
 */

void salsa20_setnonce(salsa20_ctx *ctx, const void *nonce)
{
  const octet *n = nonce;

  ctx->a[14] = LOAD32_L(n + 0);
  ctx->a[11] = LOAD32_L(n + 4);
  salsa20_seek(ctx, 0);
}

/* --- @salsa20_seek@, @salsa20_seeku64@ --- *
 *
 * Arguments:   @salsa20_ctx *ctx@ = pointer to context
 *              @unsigned long i@, @kludge64 i@ = new position to set
 *
 * Returns:     ---
 *
 * Use:         Sets a new stream position, in units of Salsa20 output
 *              blocks, which are @SALSA20_OUTSZ@ bytes each.  Byte
 *              granularity can be achieved by calling @salsa20R_encrypt@
 *              appropriately.
 */

void salsa20_seek(salsa20_ctx *ctx, unsigned long i)
  { kludge64 ii; ASSIGN64(ii, i); salsa20_seeku64(ctx, ii); }

void salsa20_seeku64(salsa20_ctx *ctx, kludge64 i)
{
  ctx->a[8] = LO64(i); ctx->a[5] = HI64(i);
  ctx->bufi = SALSA20_OUTSZ;
}

/* --- @salsa20_tell@, @salsa20_tellu64@ --- *
 *
 * Arguments:   @salsa20_ctx *ctx@ = pointer to context
 *
 * Returns:     The current position in the output stream, in blocks,
 *              rounding upwards.
 */

unsigned long salsa20_tell(salsa20_ctx *ctx)
  { kludge64 i = salsa20_tellu64(ctx); return (GET64(unsigned long, i)); }

kludge64 salsa20_tellu64(salsa20_ctx *ctx)
  { kludge64 i; SET64(i, ctx->a[5], ctx->a[8]); return (i); }

/* --- @salsa20{,12,8}_encrypt@ --- *
 *
 * Arguments:   @salsa20_ctx *ctx@ = pointer to context
 *              @const void *src@ = source buffer (or null)
 *              @void *dest@ = destination buffer (or null)
 *              @size_t sz@ = size of the buffers
 *
 * Returns:     ---
 *
 * Use:         Encrypts or decrypts @sz@ bytes of data from @src@ to @dest@.
 *              Salsa20 works by XORing plaintext with a keystream, so
 *              encryption and decryption are the same operation.  If @dest@
 *              is null then ignore @src@ and skip @sz@ bytes of the
 *              keystream.  If @src@ is null, then just write the keystream
 *              to @dest@.
 */

#define SALSA20_ENCRYPT(r, ctx, src, dest, sz)                          \
  SALSA20_DECOR(salsa20, r, _encrypt)(ctx, src, dest, sz)
#define DEFENCRYPT(r)                                                   \
  void SALSA20_ENCRYPT(r, salsa20_ctx *ctx, const void *src,            \
                       void *dest, size_t sz)                           \
  {                                                                     \
    salsa20_matrix b;                                                   \
    const octet *s = src;                                               \
    octet *d = dest;                                                    \
    size_t n;                                                           \
    kludge64 pos, delta;                                                \
                                                                        \
    SALSA20_OUTBUF(ctx, d, s, sz);                                      \
    if (!sz) return;                                                    \
                                                                        \
    if (!dest) {                                                        \
      n = sz/SALSA20_OUTSZ;                                             \
      pos = salsa20_tellu64(ctx);                                       \
      ASSIGN64(delta, n);                                               \
      ADD64(pos, pos, delta);                                           \
      salsa20_seeku64(ctx, pos);                                        \
      sz = sz%SALSA20_OUTSZ;                                            \
    } else if (!src) {                                                  \
      while (sz >= SALSA20_OUTSZ) {                                     \
        core(r, ctx->a, b);                                             \
        SALSA20_STEP(ctx->a);                                           \
        SALSA20_GENFULL(b, d);                                          \
        sz -= SALSA20_OUTSZ;                                            \
      }                                                                 \
    } else {                                                            \
      while (sz >= SALSA20_OUTSZ) {                                     \
        core(r, ctx->a, b);                                             \
        SALSA20_STEP(ctx->a);                                           \
        SALSA20_MIXFULL(b, d, s);                                       \
        sz -= SALSA20_OUTSZ;                                            \
      }                                                                 \
    }                                                                   \
                                                                        \
    if (sz) {                                                           \
      core(r, ctx->a, b);                                               \
      SALSA20_STEP(ctx->a);                                             \
      SALSA20_PREPBUF(ctx, b);                                          \
      SALSA20_OUTBUF(ctx, d, s, sz);                                    \
      assert(!sz);                                                      \
    }                                                                   \
  }
SALSA20_VARS(DEFENCRYPT)

/*----- HSalsa20 implementation -------------------------------------------*/

#define HSALSA20_RAW(r, ctx, src, dest)                                 \
  SALSA20_DECOR(hsalsa20, r, _raw)(ctx, src, dest)
#define HSALSA20_PRF(r, ctx, src, dest)                                 \
  SALSA20_DECOR(hsalsa20, r, _prf)(ctx, src, dest)

/* --- @hsalsa20{,12,8}_prf@ --- *
 *
 * Arguments:   @salsa20_ctx *ctx@ = pointer to context
 *              @const void *src@ = the input (@HSALSA20_INSZ@ bytes)
 *              @void *dest@ = the output (@HSALSA20_OUTSZ@ bytes)
 *
 * Returns:     ---
 *
 * Use:         Apply the HSalsa20/r pseudorandom function to @src@, writing
 *              the result to @out@.
 */

#define DEFHSALSA20(r)                                                  \
  static void HSALSA20_RAW(r, salsa20_matrix k,                         \
                           const uint32 *src, uint32 *dest)             \
  {                                                                     \
    salsa20_matrix a;                                                   \
    int i;                                                              \
                                                                        \
    /* --- HSalsa20, computed from full Salsa20 --- *                   \
     *                                                                  \
     * The security proof makes use of the fact that HSalsa20 (i.e.,    \
     * without the final feedforward step) can be computed from full    \
     * Salsa20 using only knowledge of the non-secret input.  I don't   \
     * want to compromise the performance of the main function by       \
     * making the feedforward step separate, but this operation is less \
     * speed critical, so we do it the harder way.                      \
     */                                                                 \
                                                                        \
    for (i = 0; i < 4; i++) k[14 - 3*i] = src[i];                       \
    core(r, k, a);                                                      \
    for (i = 0; i < 4; i++) dest[i] = a[5*i] - k[i];                    \
    for (i = 4; i < 8; i++) dest[i] = a[i + 2] - k[26 - 3*i];           \
  }                                                                     \
                                                                        \
  void HSALSA20_PRF(r, salsa20_ctx *ctx, const void *src, void *dest)   \
  {                                                                     \
    const octet *s = src;                                               \
    octet *d = dest;                                                    \
    uint32 in[4], out[8];                                               \
    int i;                                                              \
                                                                        \
    for (i = 0; i < 4; i++) in[i] = LOAD32_L(s + 4*i);                  \
    HSALSA20_RAW(r, ctx->a, in, out);                                   \
    for (i = 0; i < 8; i++) STORE32_L(d + 4*i, out[i]);                 \
  }
SALSA20_VARS(DEFHSALSA20)

/*----- XSalsa20 implementation -------------------------------------------*/

/* --- Some convenient macros for naming functions --- *
 *
 * Because the crypto core is involved in XSalsa20/r's per-nonce setup, we
 * need to take an interest in the number of rounds in most of the various
 * functions, and it will probably help if we distinguish the context
 * structures for the various versions.
 */

#define XSALSA20_CTX(r) SALSA20_DECOR(xsalsa20, r, _ctx)
#define XSALSA20_INIT(r, ctx, k, ksz, n)                                \
  SALSA20_DECOR(xsalsa20, r, _init)(ctx, k, ksz, n)
#define XSALSA20_SETNONCE(r, ctx, n)                                    \
  SALSA20_DECOR(xsalsa20, r, _setnonce)(ctx, n)
#define XSALSA20_SEEK(r, ctx, i)                                        \
  SALSA20_DECOR(xsalsa20, r, _seek)(ctx, i)
#define XSALSA20_SEEKU64(r, ctx, i)                                     \
  SALSA20_DECOR(xsalsa20, r, _seeku64)(ctx, i)
#define XSALSA20_TELL(r, ctx)                                           \
  SALSA20_DECOR(xsalsa20, r, _tell)(ctx)
#define XSALSA20_TELLU64(r, ctx)                                        \
  SALSA20_DECOR(xsalsa20, r, _tellu64)(ctx)
#define XSALSA20_ENCRYPT(r, ctx, src, dest, sz)                         \
  SALSA20_DECOR(xsalsa20, r, _encrypt)(ctx, src, dest, sz)

/* --- @xsalsa20{,12,8}_init@ --- *
 *
 * Arguments:   @xsalsa20R_ctx *ctx@ = the context to fill in
 *              @const void *key@ = pointer to key material
 *              @size_t ksz@ = size of key (either 32 or 16)
 *              @const void *nonce@ = initial nonce, or null
 *
 * Returns:     ---
 *
 * Use:         Initializes an XSalsa20/r context ready for use.
 *
 *              There is a different function for each number of rounds,
 *              unlike for plain Salsa20.
 */

#define DEFXINIT(r)                                                     \
  void XSALSA20_INIT(r, XSALSA20_CTX(r) *ctx,                           \
                        const void *key, size_t ksz, const void *nonce) \
  {                                                                     \
    static const octet zerononce[XSALSA20_NONCESZ];                     \
                                                                        \
    populate(ctx->k, key, ksz);                                         \
    ctx->s.a[ 0] = SALSA20_A256;                                        \
    ctx->s.a[ 1] = SALSA20_B256;                                        \
    ctx->s.a[ 2] = SALSA20_C256;                                        \
    ctx->s.a[ 3] = SALSA20_D256;                                        \
    XSALSA20_SETNONCE(r, ctx, nonce ? nonce : zerononce);               \
  }
SALSA20_VARS(DEFXINIT)

/* --- @xsalsa20{,12,8}_setnonce@ --- *
 *
 * Arguments:   @xsalsa20R_ctx *ctx@ = pointer to context
 *              @const void *nonce@ = the nonce (@XSALSA20_NONCESZ@ bytes)
 *
 * Returns:     ---
 *
 * Use:         Set a new nonce in the context @ctx@, e.g., for processing a
 *              different message.  The stream position is reset to zero (see
 *              @salsa20_seek@ etc.).
 *
 *              There is a different function for each number of rounds,
 *              unlike for plain Salsa20.
 */

#define DEFXNONCE(r)                                                    \
  void XSALSA20_SETNONCE(r, XSALSA20_CTX(r) *ctx, const void *nonce)    \
  {                                                                     \
    const octet *n = nonce;                                             \
    uint32 in[4], out[8];                                               \
    int i;                                                              \
                                                                        \
    for (i = 0; i < 4; i++) in[i] = LOAD32_L(n + 4*i);                  \
    HSALSA20_RAW(r, ctx->k, in, out);                                   \
    for (i = 0; i < 4; i++) ctx->s.a[13 - 3*i] = out[i];                \
    for (i = 4; i < 8; i++) ctx->s.a[27 - 3*i] = out[i];                \
    salsa20_setnonce(&ctx->s, n + 16);                                  \
  }
SALSA20_VARS(DEFXNONCE)

/* --- @xsalsa20{,12,8}_seek@, @xsalsa20{,12,8}_seeku64@ --- *
 *
 * Arguments:   @xsalsa20R_ctx *ctx@ = pointer to context
 *              @unsigned long i@, @kludge64 i@ = new position to set
 *
 * Returns:     ---
 *
 * Use:         Sets a new stream position, in units of Salsa20 output
 *              blocks, which are @XSALSA20_OUTSZ@ bytes each.  Byte
 *              granularity can be achieved by calling @xsalsa20R_encrypt@
 *              appropriately.
 *
 *              There is a different function for each number of rounds,
 *              unlike for plain Salsa20, because the context structures are
 *              different.
 */

/* --- @xsalsa20{,12,8}_tell@, @xsalsa20{,12,8}_tellu64@ --- *
 *
 * Arguments:   @salsa20_ctx *ctx@ = pointer to context
 *
 * Returns:     The current position in the output stream, in blocks,
 *              rounding upwards.
 *
 *              There is a different function for each number of rounds,
 *              unlike for plain Salsa20, because the context structures are
 *              different.
 */

/* --- @xsalsa20{,12,8}_encrypt@ --- *
 *
 * Arguments:   @xsalsa20R_ctx *ctx@ = pointer to context
 *              @const void *src@ = source buffer (or null)
 *              @void *dest@ = destination buffer (or null)
 *              @size_t sz@ = size of the buffers
 *
 * Returns:     ---
 *
 * Use:         Encrypts or decrypts @sz@ bytes of data from @src@ to @dest@.
 *              XSalsa20 works by XORing plaintext with a keystream, so
 *              encryption and decryption are the same operation.  If @dest@
 *              is null then ignore @src@ and skip @sz@ bytes of the
 *              keystream.  If @src@ is null, then just write the keystream
 *              to @dest@.
 */

#define DEFXPASSTHRU(r)                                                 \
  void XSALSA20_SEEK(r, XSALSA20_CTX(r) *ctx, unsigned long i)          \
    { salsa20_seek(&ctx->s, i); }                                       \
  void XSALSA20_SEEKU64(r, XSALSA20_CTX(r) *ctx, kludge64 i)            \
    { salsa20_seeku64(&ctx->s, i); }                                    \
  unsigned long XSALSA20_TELL(r, XSALSA20_CTX(r) *ctx)                  \
    { return salsa20_tell(&ctx->s); }                                   \
  kludge64 XSALSA20_TELLU64(r, XSALSA20_CTX(r) *ctx)                    \
    { return salsa20_tellu64(&ctx->s); }                                \
  void XSALSA20_ENCRYPT(r, XSALSA20_CTX(r) *ctx,                        \
                        const void *src, void *dest, size_t sz)         \
    { SALSA20_ENCRYPT(r, &ctx->s, src, dest, sz); }
SALSA20_VARS(DEFXPASSTHRU)

/*----- Generic cipher interface ------------------------------------------*/

typedef struct gctx { gcipher c; salsa20_ctx ctx; } gctx;

static void gsetiv(gcipher *c, const void *iv)
  { gctx *g = (gctx *)c; salsa20_setnonce(&g->ctx, iv); }

static void gdestroy(gcipher *c)
  { gctx *g = (gctx *)c; BURN(*g); S_DESTROY(g); }

#define DEFGCIPHER(r)                                                   \
                                                                        \
  static const gcipher_ops gops_##r;                                    \
                                                                        \
  static gcipher *ginit_##r(const void *k, size_t sz)                   \
  {                                                                     \
    gctx *g = S_CREATE(gctx);                                           \
    g->c.ops = &gops_##r;                                               \
    salsa20_init(&g->ctx, k, sz, 0);                                    \
    return (&g->c);                                                     \
  }                                                                     \
                                                                        \
  static void gencrypt_##r(gcipher *c, const void *s,                   \
                           void *t, size_t sz)                          \
    { gctx *g = (gctx *)c; SALSA20_ENCRYPT(r, &g->ctx, s, t, sz); }     \
                                                                        \
  static const gcipher_ops gops_##r = {                                 \
    &SALSA20_DECOR(salsa20, r, ),                                       \
    gencrypt_##r, gencrypt_##r, gdestroy, gsetiv, 0                     \
  };                                                                    \
                                                                        \
  const gccipher SALSA20_DECOR(salsa20, r, ) = {                        \
    SALSA20_NAME_##r, salsa20_keysz,                                    \
    SALSA20_NONCESZ, ginit_##r                                          \
  };

SALSA20_VARS(DEFGCIPHER)

#define DEFGXCIPHER(r)                                                  \
                                                                        \
  typedef struct { gcipher c; XSALSA20_CTX(r) ctx; } gxctx_##r;         \
                                                                        \
  static void gxsetiv_##r(gcipher *c, const void *iv)                   \
    { gxctx_##r *g = (gxctx_##r *)c; XSALSA20_SETNONCE(r, &g->ctx, iv); } \
                                                                        \
  static void gxdestroy_##r(gcipher *c)                                 \
    { gxctx_##r *g = (gxctx_##r *)c; BURN(*g); S_DESTROY(g); }          \
                                                                        \
  static const gcipher_ops gxops_##r;                                   \
                                                                        \
  static gcipher *gxinit_##r(const void *k, size_t sz)                  \
  {                                                                     \
    gxctx_##r *g = S_CREATE(gxctx_##r);                                 \
    g->c.ops = &gxops_##r;                                              \
    XSALSA20_INIT(r, &g->ctx, k, sz, 0);                                \
    return (&g->c);                                                     \
  }                                                                     \
                                                                        \
  static void gxencrypt_##r(gcipher *c, const void *s,                  \
                            void *t, size_t sz)                         \
  {                                                                     \
    gxctx_##r *g = (gxctx_##r *)c;                                      \
    XSALSA20_ENCRYPT(r, &g->ctx, s, t, sz);                             \
  }                                                                     \
                                                                        \
  static const gcipher_ops gxops_##r = {                                \
    &SALSA20_DECOR(xsalsa20, r, ),                                      \
    gxencrypt_##r, gxencrypt_##r, gxdestroy_##r, gxsetiv_##r, 0         \
  };                                                                    \
                                                                        \
  const gccipher SALSA20_DECOR(xsalsa20, r, ) = {                       \
    "x" SALSA20_NAME_##r, salsa20_keysz,                                \
    XSALSA20_NONCESZ, gxinit_##r                                        \
  };

SALSA20_VARS(DEFGXCIPHER)

/*----- Generic random number generator interface -------------------------*/

typedef struct grops {
  size_t noncesz;
  void (*seek)(void *, kludge64);
  kludge64 (*tell)(void *);
  void (*setnonce)(void *, const void *);
  void (*generate)(void *, void *, size_t);
} grops;

typedef struct grbasectx {
  grand r;
  const grops *ops;
} grbasectx;

static int grmisc(grand *r, unsigned op, ...)
{
  octet buf[XSALSA20_NONCESZ];
  grbasectx *g = (grbasectx *)r;
  grand *rr;
  const octet *p;
  size_t sz;
  uint32 i;
  unsigned long ul;
  kludge64 pos;
  va_list ap;
  int rc = 0;

  va_start(ap, op);

  switch (op) {
    case GRAND_CHECK:
      switch (va_arg(ap, unsigned)) {
        case GRAND_CHECK:
        case GRAND_SEEDINT:
        case GRAND_SEEDUINT32:
        case GRAND_SEEDBLOCK:
        case GRAND_SEEDRAND:
        case SALSA20_SEEK:
        case SALSA20_SEEKU64:
        case SALSA20_TELL:
        case SALSA20_TELLU64:
          rc = 1;
          break;
        default:
          rc = 0;
          break;
      }
      break;

    case GRAND_SEEDINT:
      i = va_arg(ap, unsigned); STORE32_L(buf, i);
      memset(buf + 4, 0, g->ops->noncesz - 4);
      g->ops->setnonce(g, buf);
      break;
    case GRAND_SEEDUINT32:
      i = va_arg(ap, uint32); STORE32_L(buf, i);
      memset(buf + 4, 0, g->ops->noncesz - 4);
      g->ops->setnonce(g, buf);
      break;
    case GRAND_SEEDBLOCK:
      p = va_arg(ap, const void *);
      sz = va_arg(ap, size_t);
      if (sz < g->ops->noncesz) {
        memcpy(buf, p, sz);
        memset(buf + sz, 0, g->ops->noncesz - sz);
        p = buf;
      }
      g->ops->setnonce(g, p);
      break;
    case GRAND_SEEDRAND:
      rr = va_arg(ap, grand *);
      rr->ops->fill(rr, buf, g->ops->noncesz);
      g->ops->setnonce(g, buf);
      break;
    case SALSA20_SEEK:
      ul = va_arg(ap, unsigned long); ASSIGN64(pos, ul);
      g->ops->seek(g, pos);
      break;
    case SALSA20_SEEKU64:
      pos = va_arg(ap, kludge64);
      g->ops->seek(g, pos);
      break;
    case SALSA20_TELL:
      pos = g->ops->tell(g);
      *va_arg(ap, unsigned long *) = GET64(unsigned long, pos);
      break;
    case SALSA20_TELLU64:
      *va_arg(ap, kludge64 *) = g->ops->tell(g);
      break;
    default:
      GRAND_BADOP;
      break;
  }

  return (rc);
}

static octet grbyte(grand *r)
{
  grbasectx *g = (grbasectx *)r;
  octet o;
  g->ops->generate(g, &o, 1);
  return (o);
}

static uint32 grword(grand *r)
{
  grbasectx *g = (grbasectx *)r;
  octet b[4];
  g->ops->generate(g, b, sizeof(b));
  return (LOAD32_L(b));
}

static void grfill(grand *r, void *p, size_t sz)
{
  grbasectx *g = (grbasectx *)r;
  g->ops->generate(r, p, sz);
}

typedef struct grctx {
  grbasectx r;
  salsa20_ctx ctx;
} grctx;

static void gr_seek(void *r, kludge64 pos)
  { grctx *g = r; salsa20_seeku64(&g->ctx, pos); }

static kludge64 gr_tell(void *r)
  { grctx *g = r; return (salsa20_tellu64(&g->ctx)); }

static void gr_setnonce(void *r, const void *n)
  { grctx *g = r; salsa20_setnonce(&g->ctx, n); }

static void grdestroy(grand *r)
  { grctx *g = (grctx *)r; BURN(*g); S_DESTROY(g); }

#define DEFGRAND(rr)                                                    \
                                                                        \
  static void gr_generate_##rr(void *r, void *b, size_t sz)             \
    { grctx *g = r; SALSA20_ENCRYPT(rr, &g->ctx, 0, b, sz); }           \
                                                                        \
  static const grops grops_##rr =                                       \
    { SALSA20_NONCESZ, gr_seek, gr_tell,                                \
      gr_setnonce, gr_generate_##rr };                                  \
                                                                        \
  static const grand_ops grops_rand_##rr = {                            \
    SALSA20_NAME_##rr, GRAND_CRYPTO, 0,                                 \
    grmisc, grdestroy, grword,                                          \
    grbyte, grword, grand_range, grfill                                 \
  };                                                                    \
                                                                        \
  grand *SALSA20_DECOR(salsa20, rr, _rand)                              \
    (const void *k, size_t ksz, const void *n)                          \
  {                                                                     \
    grctx *g = S_CREATE(g);                                             \
    g->r.r.ops = &grops_rand_##rr;                                      \
    g->r.ops = &grops_##rr;                                             \
    salsa20_init(&g->ctx, k, ksz, n);                                   \
    return (&g->r.r);                                                   \
  }
SALSA20_VARS(DEFGRAND)

#define DEFXGRAND(rr)                                                   \
                                                                        \
  typedef struct grxctx_##rr {                                          \
    grbasectx r;                                                        \
    XSALSA20_CTX(rr) ctx;                                               \
  } grxctx_##rr;                                                        \
                                                                        \
  static void grx_seek_##rr(void *r, kludge64 pos)                      \
    { grxctx_##rr *g = r; XSALSA20_SEEKU64(rr, &g->ctx, pos); }         \
                                                                        \
  static kludge64 grx_tell_##rr(void *r)                                \
    { grxctx_##rr *g = r; return (XSALSA20_TELLU64(rr, &g->ctx)); }     \
                                                                        \
  static void grx_setnonce_##rr(void *r, const void *n)                 \
    { grxctx_##rr *g = r; XSALSA20_SETNONCE(rr, &g->ctx, n); }          \
                                                                        \
  static void grxdestroy_##rr(grand *r)                                 \
    { grxctx_##rr *g = (grxctx_##rr *)r; BURN(*g); S_DESTROY(g); }      \
                                                                        \
  static void grx_generate_##rr(void *r, void *b, size_t sz)            \
    { grxctx_##rr *g = r; XSALSA20_ENCRYPT(rr, &g->ctx, 0, b, sz); }    \
                                                                        \
  static const grops grxops_##rr =                                      \
  { XSALSA20_NONCESZ, grx_seek_##rr, grx_tell_##rr,                     \
      grx_setnonce_##rr, grx_generate_##rr };                           \
                                                                        \
  static const grand_ops grxops_rand_##rr = {                           \
    "x" SALSA20_NAME_##rr, GRAND_CRYPTO, 0,                             \
    grmisc, grxdestroy_##rr, grword,                                    \
    grbyte, grword, grand_range, grfill                                 \
  };                                                                    \
                                                                        \
  grand *SALSA20_DECOR(xsalsa20, rr, _rand)                             \
    (const void *k, size_t ksz, const void *n)                          \
  {                                                                     \
    grxctx_##rr *g = S_CREATE(g);                                       \
    g->r.r.ops = &grxops_rand_##rr;                                     \
    g->r.ops = &grxops_##rr;                                            \
    XSALSA20_INIT(rr, &g->ctx, k, ksz, n);                              \
    return (&g->r.r);                                                   \
  }
SALSA20_VARS(DEFXGRAND)

/*----- Test rig ----------------------------------------------------------*/

#ifdef TEST_RIG

#include <stdio.h>
#include <string.h>

#include <mLib/quis.h>
#include <mLib/testrig.h>

static const int perm[] = {
   0, 13, 10,  7,
   4,  1, 14, 11,
   8,  5,  2, 15,
  12,  9,  6,  3
};

#define DEFVCORE(r)                                                     \
  static int v_core_##r(dstr *v)                                        \
  {                                                                     \
    salsa20_matrix a, b;                                                \
    dstr d = DSTR_INIT;                                                 \
    int i, j, n;                                                        \
    int ok = 1;                                                         \
                                                                        \
    DENSURE(&d, SALSA20_OUTSZ); d.len = SALSA20_OUTSZ;                  \
    n = *(int *)v[0].buf;                                               \
    for (i = 0; i < SALSA20_OUTSZ/4; i++)                               \
      b[i] = LOAD32_L(v[1].buf + 4*i);                                  \
    for (i = 0; i < n; i++) {                                           \
      for (j = 0; j < 16; j++) a[perm[j]] = b[j];                       \
      core(r, a, b);                                                    \
      memcpy(a, b, sizeof(a));                                          \
    }                                                                   \
    for (i = 0; i < SALSA20_OUTSZ/4; i++) STORE32_L(d.buf + 4*i, b[i]); \
                                                                        \
    if (d.len != v[2].len || memcmp(d.buf, v[2].buf, v[2].len) != 0) {  \
      ok = 0;                                                           \
      printf("\nfail core:"                                             \
             "\n\titerations = %d"                                      \
             "\n\tin       = ", n);                                     \
      type_hex.dump(&v[1], stdout);                                     \
      printf("\n\texpected   = ");                                      \
      type_hex.dump(&v[2], stdout);                                     \
      printf("\n\tcalculated = ");                                      \
      type_hex.dump(&d, stdout);                                        \
      putchar('\n');                                                    \
    }                                                                   \
                                                                        \
    dstr_destroy(&d);                                                   \
    return (ok);                                                        \
  }
SALSA20_VARS(DEFVCORE)

#define SALSA20_CTX(r) salsa20_ctx
#define SALSA20_INIT(r, ctx, k, ksz, n) salsa20_init(ctx, k, ksz, n)
#define SALSA20_SEEKU64(r, ctx, i) salsa20_seeku64(ctx, i)

#define DEFxVENC(base, BASE, r)                                         \
  static int v_encrypt_##base##_##r(dstr *v)                            \
  {                                                                     \
    BASE##_CTX(r) ctx;                                                  \
    dstr d = DSTR_INIT;                                                 \
    kludge64 pos;                                                       \
    const octet *p, *p0;                                                \
    octet *q;                                                           \
    size_t sz, sz0, step;                                               \
    unsigned long skip;                                                 \
    int ok = 1;                                                         \
                                                                        \
    if (v[4].len) { p0 = (const octet *)v[4].buf; sz0 = v[4].len; }     \
    else { p0 = 0; sz0 = v[5].len; }                                    \
    DENSURE(&d, sz0); d.len = sz0;                                      \
    skip = *(unsigned long *)v[3].buf;                                  \
                                                                        \
    step = 0;                                                           \
    while (step < sz0 + skip) {                                         \
      step = step ? 3*step + 4 : 1;                                     \
      if (step > sz0 + skip) step = sz0 + skip;                         \
      BASE##_INIT(r, &ctx, v[0].buf, v[0].len, v[1].buf);               \
      if (v[2].len) {                                                   \
        LOAD64_(pos, v[2].buf);                                         \
        BASE##_SEEKU64(r, &ctx, pos);                                   \
      }                                                                 \
                                                                        \
      for (sz = skip; sz >= step; sz -= step)                           \
        BASE##_ENCRYPT(r, &ctx, 0, 0, step);                            \
      if (sz) BASE##_ENCRYPT(r, &ctx, 0, 0, sz);                        \
      for (p = p0, q = (octet *)d.buf, sz = sz0;                        \
           sz >= step;                                                  \
           sz -= step, q += step) {                                     \
        BASE##_ENCRYPT(r, &ctx, p, q, step);                            \
        if (p) p += step;                                               \
      }                                                                 \
      if (sz) BASE##_ENCRYPT(r, &ctx, p, q, sz);                        \
                                                                        \
      if (d.len != v[5].len || memcmp(d.buf, v[5].buf, v[5].len) != 0) { \
        ok = 0;                                                         \
        printf("\nfail encrypt:"                                        \
               "\n\tstep           = %lu"                               \
               "\n\tkey    = ", (unsigned long)step);                   \
        type_hex.dump(&v[0], stdout);                                   \
        printf("\n\tnonce          = ");                                \
        type_hex.dump(&v[1], stdout);                                   \
        printf("\n\tposition   = ");                                    \
        type_hex.dump(&v[2], stdout);                                   \
        printf("\n\tskip           = %lu", skip);                       \
        printf("\n\tmessage    = ");                                    \
        type_hex.dump(&v[4], stdout);                                   \
        printf("\n\texpected   = ");                                    \
        type_hex.dump(&v[5], stdout);                                   \
        printf("\n\tcalculated = ");                                    \
        type_hex.dump(&d, stdout);                                      \
        putchar('\n');                                                  \
      }                                                                 \
    }                                                                   \
                                                                        \
    dstr_destroy(&d);                                                   \
    return (ok);                                                        \
  }
#define DEFVENC(r) DEFxVENC(salsa20, SALSA20, r)
#define DEFXVENC(r) DEFxVENC(xsalsa20, XSALSA20, r)
SALSA20_VARS(DEFVENC)
SALSA20_VARS(DEFXVENC)

static test_chunk defs[] = {
#define DEFxTAB(pre, base, r)                                           \
  { pre SALSA20_NAME_##r, v_encrypt_##base##_##r,                       \
    { &type_hex, &type_hex, &type_hex, &type_ulong,                     \
      &type_hex, &type_hex, 0 } },
#define DEFTAB(r)                                                       \
  { SALSA20_NAME_##r "-core", v_core_##r,                               \
    { &type_int, &type_hex, &type_hex, 0 } },                           \
  DEFxTAB("", salsa20, r)
#define DEFXTAB(r) DEFxTAB("x", xsalsa20, r)
SALSA20_VARS(DEFTAB)
SALSA20_VARS(DEFXTAB)
  { 0, 0, { 0 } }
};

int main(int argc, char *argv[])
{
  test_run(argc, argv, defs, SRCDIR"/t/salsa20");
  return (0);
}

#endif

/*----- That's all, folks -------------------------------------------------*/