Merge branch '2.4.x' into 2.5.x

[catacomb] / symm / chacha.c

diff --git a/symm/chacha.c b/symm/chacha.c
index f70ee9f..90a4c67 100644 (file)
--- a/symm/chacha.c
+++ b/symm/chacha.c
@@ -41,6 +41,7 @@
 #include "grand.h"
 #include "keysz.h"
 #include "paranoia.h"
+#include "rsvr.h"
 
 /*----- Global variables --------------------------------------------------*/
 
@@ -72,6 +73,7 @@ static void simple_core(unsigned r, const chacha_matrix src,
 
 #if CPUFAM_X86 || CPUFAM_AMD64
 extern core__functype chacha_core_x86ish_sse2;
+extern core__functype chacha_core_x86ish_avx;
 #endif
 
 #if CPUFAM_ARMEL
@@ -85,6 +87,8 @@ extern core__functype chacha_core_arm64;
 static core__functype *pick_core(void)
 {
 #if CPUFAM_X86 || CPUFAM_AMD64
+  DISPATCH_PICK_COND(chacha_core, chacha_core_x86ish_avx,
+                    cpu_feature_p(CPUFEAT_X86_AVX));
   DISPATCH_PICK_COND(chacha_core, chacha_core_x86ish_sse2,
                     cpu_feature_p(CPUFEAT_X86_SSE2));
 #endif
@@ -150,6 +154,8 @@ static void populate(chacha_matrix a, const void *key, size_t ksz)
 
 /*----- ChaCha implementation ---------------------------------------------*/
 
+static const octet zerononce[XCHACHA_NONCESZ];
+
 /* --- @chacha_init@ --- *
  *
  * Arguments:  @chacha_ctx *ctx@ = context to fill in
@@ -165,8 +171,6 @@ static void populate(chacha_matrix a, const void *key, size_t ksz)
 void chacha_init(chacha_ctx *ctx, const void *key, size_t ksz,
                  const void *nonce)
 {
-  static const octet zerononce[CHACHA_NONCESZ];
-
   populate(ctx->a, key, ksz);
   chacha_setnonce(ctx, nonce ? nonce : zerononce);
 }
@@ -222,7 +226,7 @@ void chacha_seek(chacha_ctx *ctx, unsigned long i)
 void chacha_seeku64(chacha_ctx *ctx, kludge64 i)
 {
   ctx->a[12] = LO64(i); ctx->a[13] = HI64(i);
-  ctx->bufi = CHACHA_OUTSZ;
+  ctx->off = CHACHA_OUTSZ;
 }
 
 void chacha_seek_ietf(chacha_ctx *ctx, uint32 i)
@@ -262,6 +266,8 @@ uint32 chacha_tell_ietf(chacha_ctx *ctx)
  *             to @dest@.
  */
 
+static const rsvr_policy policy = { 0, CHACHA_OUTSZ, CHACHA_OUTSZ };
+
 #define CHACHA_ENCRYPT(r, ctx, src, dest, sz)                          \
   chacha##r##_encrypt(ctx, src, dest, sz)
 #define DEFENCRYPT(r)                                                  \
@@ -271,41 +277,40 @@ uint32 chacha_tell_ietf(chacha_ctx *ctx)
     chacha_matrix b;                                                   \
     const octet *s = src;                                              \
     octet *d = dest;                                                   \
-    size_t n;                                                          \
+    rsvr_plan plan;                                                    \
     kludge64 pos, delta;                                               \
                                                                        \
-    SALSA20_OUTBUF(ctx, d, s, sz);                                     \
-    if (!sz) return;                                                   \
-                                                                       \
-    if (!dest) {                                                       \
-      n = sz/CHACHA_OUTSZ;                                             \
-      pos = chacha_tellu64(ctx);                                       \
-      ASSIGN64(delta, n);                                              \
-      ADD64(pos, pos, delta);                                          \
-      chacha_seeku64(ctx, pos);                                                \
-      sz = sz%CHACHA_OUTSZ;                                            \
-    } else if (!src) {                                                 \
-      while (sz >= CHACHA_OUTSZ) {                                     \
-       core(r, ctx->a, b);                                             \
-       CHACHA_STEP(ctx->a);                                            \
-       SALSA20_GENFULL(b, d);                                          \
-       sz -= CHACHA_OUTSZ;                                             \
+    rsvr_mkplan(&plan, &policy, ctx->off, sz);                         \
+                                                                       \
+    if (plan.head) {                                                   \
+      if (!ctx->off) {                                                 \
+       core(r, ctx->a, b); CHACHA_STEP(ctx->a);                        \
+       SALSA20_PREPBUF(ctx, b);                                        \
       }                                                                        \
-    } else {                                                           \
-      while (sz >= CHACHA_OUTSZ) {                                     \
-       core(r, ctx->a, b);                                             \
-       CHACHA_STEP(ctx->a);                                            \
-       SALSA20_MIXFULL(b, d, s);                                       \
-       sz -= CHACHA_OUTSZ;                                             \
+      SALSA20_OUTBUF(ctx, d, s, plan.head);                            \
+    }                                                                  \
+                                                                       \
+    ctx->off -= plan.from_rsvr;                                                \
+                                                                       \
+    if (!d) {                                                          \
+      if (plan.from_input) {                                           \
+       pos = chacha_tellu64(ctx);                                      \
+       ASSIGN64(delta, plan.from_input/SALSA20_OUTSZ);                 \
+       ADD64(pos, pos, delta);                                         \
+       chacha_seeku64(ctx, pos);                                       \
       }                                                                        \
+    } else if (!s) while (plan.from_input) {                           \
+      core(r, ctx->a, b); CHACHA_STEP(ctx->a);                         \
+      SALSA20_GENFULL(b, d); plan.from_input -= CHACHA_OUTSZ;          \
+    } else while (plan.from_input) {                                   \
+      core(r, ctx->a, b); CHACHA_STEP(ctx->a);                         \
+      SALSA20_MIXFULL(b, d, s); plan.from_input -= CHACHA_OUTSZ;       \
     }                                                                  \
                                                                        \
-    if (sz) {                                                          \
-      core(r, ctx->a, b);                                              \
-      CHACHA_STEP(ctx->a);                                             \
+    if (plan.tail) {                                                   \
+      core(r, ctx->a, b); CHACHA_STEP(ctx->a);                         \
       SALSA20_PREPBUF(ctx, b);                                         \
-      SALSA20_OUTBUF(ctx, d, s, sz);                                   \
-      assert(!sz);                                                     \
+      SALSA20_OUTBUF(ctx, d, s, plan.tail);                            \
     }                                                                  \
   }
 CHACHA_VARS(DEFENCRYPT)
@@ -401,8 +406,6 @@ CHACHA_VARS(DEFHCHACHA)
   void XCHACHA_INIT(r, XCHACHA_CTX(r) *ctx,                            \
                    const void *key, size_t ksz, const void *nonce)     \
   {                                                                    \
-    static const octet zerononce[XCHACHA_NONCESZ];                     \
-                                                                       \
     populate(ctx->k, key, ksz);                                                \
     ctx->s.a[ 0] = CHACHA_A256;                                                \
     ctx->s.a[ 1] = CHACHA_B256;                                                \