A number of small bug fixes, some motivated by compiler warnings.

[u/mdw/catacomb] / oaep.c

/* -*-c-*-
 *
 * $Id: oaep.c,v 1.6 2004/04/08 01:36:15 mdw Exp $
 *
 * Optimal asymmetric encryption packing
 *
 * (c) 2000 Straylight/Edgeware
 */

/*----- Licensing notice --------------------------------------------------*
 *
 * This file is part of Catacomb.
 *
 * Catacomb is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Library General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * Catacomb is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with Catacomb; if not, write to the Free
 * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
 * MA 02111-1307, USA.
 */

/*----- Header files ------------------------------------------------------*/

#include <string.h>

#include <mLib/alloc.h>
#include <mLib/bits.h>
#include <mLib/dstr.h>

#include "gcipher.h"
#include "ghash.h"
#include "grand.h"
#include "rsa.h"

/*----- Main code ---------------------------------------------------------*/

/* --- @oaep_encode@ --- *
 *
 * Arguments:	@mp *d@ = where to put the answer
 *		@const void *m@ = pointer to message data
 *		@size_t msz@ = size of message data
 *		@octet *b@ = spare buffer
 *		@size_t sz@ = size of the buffer (big enough)
 *		@unsigned long nbits@ = length of bits of @n@
 *		@void *p@ = pointer to OAEP parameter block
 *
 * Returns:	The encoded plaintext, or null on failure.
 *
 * Use:		Implements the operation @EME-OAEP-ENCODE@, as defined in
 *		PKCS#1 v. 2.0 (RFC2437).
 */

mp *oaep_encode(mp *d, const void *m, size_t msz, octet *b, size_t sz,
		unsigned long nbits, void *p)
{
  oaep *o = p;
  size_t hsz = o->ch->hashsz;
  ghash *h;
  octet *q, *mq, *qq;
  octet *pp;
  gcipher *c;
  size_t n;

  /* --- Ensure that everything is sensibly sized --- */

  if (2 * hsz + 2 + msz > sz)
    return (0);

  /* --- Make the `seed' value --- */

  q = b;
  *q++ = 0; sz--;
  mq = q + hsz;
  qq = q + sz;
  GR_FILL(o->r, q, hsz);

  /* --- Fill in the rest of the buffer --- */

  h = GH_INIT(o->ch);
  GH_HASH(h, o->ep, o->epsz);
  GH_DONE(h, mq);
  GH_DESTROY(h);
  pp = mq + hsz;
  n = sz - 2 * hsz - msz - 1;
  memset(pp, 0, n);
  pp += n;
  *pp++ = 1;
  memcpy(pp, m, msz);

  /* --- Do the packing --- */

  n = sz - hsz;
  c = GC_INIT(o->cc, q, hsz);
  GC_ENCRYPT(c, mq, mq, n);
  GC_DESTROY(c);

  c = GC_INIT(o->cc, mq, n);
  GC_ENCRYPT(c, q, q, hsz);
  GC_DESTROY(c);

  /* --- Done --- */

  return (mp_loadb(d, b, sz + 1));
}

/* --- @oaep_decode@ --- *
 *
 * Arguments:	@mp *m@ = the decrypted message
 *		@octet *b@ = pointer to a buffer to work in
 *		@size_t sz@ = the size of the buffer (big enough)
 *		@unsigned long nbits@ = the number of bits in @n@
 *		@void *p@ = pointer to OAEP parameter block
 *
 * Returns:	The length of the output string if successful, negative on
 *		failure.
 *
 * Use:		Implements the operation @EME-OAEP-DECODE@, as defined in
 *		PKCS#1 v. 2.0 (RFC2437).
 */

static int memeq(const void *xx, const void *yy, size_t sz)
{
  int eq = 1;
  const octet *x = xx, *y = yy;
  while (sz) {				/* Always check every byte */
    if (*x++ != *y++) eq = 0;
    sz--;
  }
  return (eq);
}

int oaep_decode(mp *m, octet *b, size_t sz, unsigned long nbits, void *p)
{
  oaep *o = p;
  gcipher *c;
  ghash *h;
  octet *q, *mq, *qq;
  octet *pp;
  unsigned bad = 0;
  size_t n;
  size_t hsz = o->ch->hashsz;

  /* --- Ensure that the block is large enough --- */

  if (sz < 2 * hsz)			/* Doesn't depend on ciphertext */
    return (-1);

  /* --- Decrypt the message --- */

  mp_storeb(m, b, sz);
  q = b;
  bad = *q;
  q++; sz--;
  mq = q + hsz;
  qq = q + sz;
  n = sz - hsz;
  c = GC_INIT(o->cc, mq, n);
  GC_DECRYPT(c, q, q, hsz);
  GC_DESTROY(c);

  c = GC_INIT(o->cc, q, hsz);
  GC_DECRYPT(c, mq, mq, n);
  GC_DESTROY(c);
  q--;

  /* --- Check the hash on the encoding parameters --- */

  h = GH_INIT(o->ch);
  GH_HASH(h, o->ep, o->epsz);
  GH_DONE(h, q);
  GH_DESTROY(h);
  bad |= !memeq(q, mq, hsz);

  /* --- Now find the start of the actual message --- */

  pp = mq + hsz;
  while (*pp == 0 && pp < qq)
    pp++;
  bad |= (pp >= qq) | (*pp != 1);
  pp++;
  n = qq - pp;
  memmove(q, pp, n);
  return (bad ? -1 : n);
}

/*----- That's all, folks -------------------------------------------------*/