catcrypt.c: Don't close output file twice.
[u/mdw/catacomb] / oaep.c
1 /* -*-c-*-
2 *
3 * $Id: oaep.c,v 1.6 2004/04/08 01:36:15 mdw Exp $
4 *
5 * Optimal asymmetric encryption packing
6 *
7 * (c) 2000 Straylight/Edgeware
8 */
9
10 /*----- Licensing notice --------------------------------------------------*
11 *
12 * This file is part of Catacomb.
13 *
14 * Catacomb is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU Library General Public License as
16 * published by the Free Software Foundation; either version 2 of the
17 * License, or (at your option) any later version.
18 *
19 * Catacomb is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU Library General Public License for more details.
23 *
24 * You should have received a copy of the GNU Library General Public
25 * License along with Catacomb; if not, write to the Free
26 * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
27 * MA 02111-1307, USA.
28 */
29
30 /*----- Header files ------------------------------------------------------*/
31
32 #include <string.h>
33
34 #include <mLib/alloc.h>
35 #include <mLib/bits.h>
36 #include <mLib/dstr.h>
37
38 #include "gcipher.h"
39 #include "ghash.h"
40 #include "grand.h"
41 #include "rsa.h"
42
43 /*----- Main code ---------------------------------------------------------*/
44
45 /* --- @oaep_encode@ --- *
46 *
47 * Arguments: @mp *d@ = where to put the answer
48 * @const void *m@ = pointer to message data
49 * @size_t msz@ = size of message data
50 * @octet *b@ = spare buffer
51 * @size_t sz@ = size of the buffer (big enough)
52 * @unsigned long nbits@ = length of bits of @n@
53 * @void *p@ = pointer to OAEP parameter block
54 *
55 * Returns: The encoded plaintext, or null on failure.
56 *
57 * Use: Implements the operation @EME-OAEP-ENCODE@, as defined in
58 * PKCS#1 v. 2.0 (RFC2437).
59 */
60
61 mp *oaep_encode(mp *d, const void *m, size_t msz, octet *b, size_t sz,
62 unsigned long nbits, void *p)
63 {
64 oaep *o = p;
65 size_t hsz = o->ch->hashsz;
66 ghash *h;
67 octet *q, *mq, *qq;
68 octet *pp;
69 gcipher *c;
70 size_t n;
71
72 /* --- Ensure that everything is sensibly sized --- */
73
74 if (2 * hsz + 2 + msz > sz)
75 return (0);
76
77 /* --- Make the `seed' value --- */
78
79 q = b;
80 *q++ = 0; sz--;
81 mq = q + hsz;
82 qq = q + sz;
83 GR_FILL(o->r, q, hsz);
84
85 /* --- Fill in the rest of the buffer --- */
86
87 h = GH_INIT(o->ch);
88 GH_HASH(h, o->ep, o->epsz);
89 GH_DONE(h, mq);
90 GH_DESTROY(h);
91 pp = mq + hsz;
92 n = sz - 2 * hsz - msz - 1;
93 memset(pp, 0, n);
94 pp += n;
95 *pp++ = 1;
96 memcpy(pp, m, msz);
97
98 /* --- Do the packing --- */
99
100 n = sz - hsz;
101 c = GC_INIT(o->cc, q, hsz);
102 GC_ENCRYPT(c, mq, mq, n);
103 GC_DESTROY(c);
104
105 c = GC_INIT(o->cc, mq, n);
106 GC_ENCRYPT(c, q, q, hsz);
107 GC_DESTROY(c);
108
109 /* --- Done --- */
110
111 return (mp_loadb(d, b, sz + 1));
112 }
113
114 /* --- @oaep_decode@ --- *
115 *
116 * Arguments: @mp *m@ = the decrypted message
117 * @octet *b@ = pointer to a buffer to work in
118 * @size_t sz@ = the size of the buffer (big enough)
119 * @unsigned long nbits@ = the number of bits in @n@
120 * @void *p@ = pointer to OAEP parameter block
121 *
122 * Returns: The length of the output string if successful, negative on
123 * failure.
124 *
125 * Use: Implements the operation @EME-OAEP-DECODE@, as defined in
126 * PKCS#1 v. 2.0 (RFC2437).
127 */
128
129 static int memeq(const void *xx, const void *yy, size_t sz)
130 {
131 int eq = 1;
132 const octet *x = xx, *y = yy;
133 while (sz) { /* Always check every byte */
134 if (*x++ != *y++) eq = 0;
135 sz--;
136 }
137 return (eq);
138 }
139
140 int oaep_decode(mp *m, octet *b, size_t sz, unsigned long nbits, void *p)
141 {
142 oaep *o = p;
143 gcipher *c;
144 ghash *h;
145 octet *q, *mq, *qq;
146 octet *pp;
147 unsigned bad = 0;
148 size_t n;
149 size_t hsz = o->ch->hashsz;
150
151 /* --- Ensure that the block is large enough --- */
152
153 if (sz < 2 * hsz) /* Doesn't depend on ciphertext */
154 return (-1);
155
156 /* --- Decrypt the message --- */
157
158 mp_storeb(m, b, sz);
159 q = b;
160 bad = *q;
161 q++; sz--;
162 mq = q + hsz;
163 qq = q + sz;
164 n = sz - hsz;
165 c = GC_INIT(o->cc, mq, n);
166 GC_DECRYPT(c, q, q, hsz);
167 GC_DESTROY(c);
168
169 c = GC_INIT(o->cc, q, hsz);
170 GC_DECRYPT(c, mq, mq, n);
171 GC_DESTROY(c);
172 q--;
173
174 /* --- Check the hash on the encoding parameters --- */
175
176 h = GH_INIT(o->ch);
177 GH_HASH(h, o->ep, o->epsz);
178 GH_DONE(h, q);
179 GH_DESTROY(h);
180 bad |= !memeq(q, mq, hsz);
181
182 /* --- Now find the start of the actual message --- */
183
184 pp = mq + hsz;
185 while (*pp == 0 && pp < qq)
186 pp++;
187 bad |= (pp >= qq) | (*pp != 1);
188 pp++;
189 n = qq - pp;
190 memmove(q, pp, n);
191 return (bad ? -1 : n);
192 }
193
194 /*----- That's all, folks -------------------------------------------------*/