Commit | Line | Data |
---|---|---|
9e6a4409 MW |
1 | /// -*- mode: asm; asm-comment-char: ?/ -*- |
2 | /// | |
3 | /// GCM acceleration for ARM64 processors | |
4 | /// | |
5 | /// (c) 2019 Straylight/Edgeware | |
6 | /// | |
7 | ||
8 | ///----- Licensing notice --------------------------------------------------- | |
9 | /// | |
10 | /// This file is part of Catacomb. | |
11 | /// | |
12 | /// Catacomb is free software: you can redistribute it and/or modify it | |
13 | /// under the terms of the GNU Library General Public License as published | |
14 | /// by the Free Software Foundation; either version 2 of the License, or | |
15 | /// (at your option) any later version. | |
16 | /// | |
17 | /// Catacomb is distributed in the hope that it will be useful, but | |
18 | /// WITHOUT ANY WARRANTY; without even the implied warranty of | |
19 | /// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
20 | /// Library General Public License for more details. | |
21 | /// | |
22 | /// You should have received a copy of the GNU Library General Public | |
23 | /// License along with Catacomb. If not, write to the Free Software | |
24 | /// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, | |
25 | /// USA. | |
26 | ||
27 | ///-------------------------------------------------------------------------- | |
28 | /// Preliminaries. | |
29 | ||
30 | #include "config.h" | |
31 | #include "asm-common.h" | |
32 | ||
33 | .arch armv8-a+crypto | |
34 | ||
35 | .text | |
36 | ||
37 | ///-------------------------------------------------------------------------- | |
38 | /// Multiplication macros. | |
39 | ||
40 | // The good news is that we have a fancy instruction to do the | |
41 | // multiplications. The bad news is that it's not particularly well- | |
42 | // suited to the job. | |
43 | // | |
44 | // For one thing, it only does a 64-bit multiplication, so in general | |
45 | // we'll need to synthesize the full-width multiply by hand. For | |
46 | // another thing, it doesn't help with the reduction, so we have to | |
47 | // do that by hand too. And, finally, GCM has crazy bit ordering, | |
48 | // and the instruction does nothing useful for that at all. | |
49 | // | |
50 | // Focusing on that last problem first: the bits aren't in monotonic | |
51 | // significance order unless we permute them. Fortunately, ARM64 has | |
52 | // an instruction which will just permute the bits in each byte for | |
53 | // us, so we don't have to worry about this very much. | |
54 | // | |
55 | // Our main weapons, the `pmull' and `pmull2' instructions, work on | |
56 | // 64-bit operands, in half of a vector register, and produce 128-bit | |
57 | // results. But neither of them will multiply the high half of one | |
58 | // vector by the low half of a second one, so we have a problem, | |
59 | // which we solve by representing one of the operands redundantly: | |
60 | // rather than packing the 64-bit pieces together, we duplicate each | |
61 | // 64-bit piece across both halves of a register. | |
62 | // | |
63 | // The commentary for `mul128' is the most detailed. The other | |
64 | // macros assume that you've already read and understood that. | |
65 | ||
66 | .macro mul128 | |
67 | // Enter with u and v in v0 and v1/v2 respectively, and 0 in v31; | |
68 | // leave with z = u v in v0. Clobbers v1--v6. | |
69 | ||
70 | // First for the double-precision multiplication. It's tempting to | |
71 | // use Karatsuba's identity here, but I suspect that loses more in | |
72 | // the shifting, bit-twiddling, and dependency chains that it gains | |
73 | // in saving a multiplication which otherwise pipelines well. | |
74 | // v0 = // (u_0; u_1) | |
75 | // v1/v2 = // (v_0; v_1) | |
76 | pmull2 v3.1q, v0.2d, v1.2d // u_1 v_0 | |
77 | pmull v4.1q, v0.1d, v2.1d // u_0 v_1 | |
78 | pmull2 v5.1q, v0.2d, v2.2d // (t_1; x_3) = u_1 v_1 | |
79 | pmull v6.1q, v0.1d, v1.1d // (x_0; t_0) = u_0 v_0 | |
80 | ||
81 | // Arrange the pieces to form a double-precision polynomial. | |
82 | eor v3.16b, v3.16b, v4.16b // (m_0; m_1) = u_0 v_1 + u_1 v_0 | |
83 | vshr128 v4, v3, 64 // (m_1; 0) | |
84 | vshl128 v3, v3, 64 // (0; m_0) | |
85 | eor v1.16b, v5.16b, v4.16b // (x_2; x_3) | |
86 | eor v0.16b, v6.16b, v3.16b // (x_0; x_1) | |
87 | ||
88 | // And now the only remaining difficulty is that the result needs to | |
89 | // be reduced modulo p(t) = t^128 + t^7 + t^2 + t + 1. Let R = t^128 | |
90 | // = t^7 + t^2 + t + 1 in our field. So far, we've calculated z_0 | |
91 | // and z_1 such that z_0 + z_1 R = u v using the identity R = t^128: | |
92 | // now we must collapse the two halves of y together using the other | |
93 | // identity R = t^7 + t^2 + t + 1. | |
94 | // | |
95 | // We do this by working on y_2 and y_3 separately, so consider y_i | |
96 | // for i = 2 or 3. Certainly, y_i t^{64i} = y_i R t^{64(i-2) = | |
97 | // (t^7 + t^2 + t + 1) y_i t^{64(i-2)}, but we can't use that | |
98 | // directly without breaking up the 64-bit word structure. Instead, | |
99 | // we start by considering just y_i t^7 t^{64(i-2)}, which again | |
100 | // looks tricky. Now, split y_i = a_i + t^57 b_i, with deg a_i < 57; | |
101 | // then | |
102 | // | |
103 | // y_i t^7 t^{64(i-2)} = a_i t^7 t^{64(i-2)} + b_i t^{64(i-1)} | |
104 | // | |
105 | // We can similarly decompose y_i t^2 and y_i t into a pair of 64-bit | |
106 | // contributions to the t^{64(i-2)} and t^{64(i-1)} words, but the | |
107 | // splits are different. This is lovely, with one small snag: when | |
108 | // we do this to y_3, we end up with a contribution back into the | |
109 | // t^128 coefficient word. But notice that only the low seven bits | |
110 | // of this word are affected, so there's no knock-on contribution | |
111 | // into the t^64 word. Therefore, if we handle the high bits of each | |
112 | // word together, and then the low bits, everything will be fine. | |
113 | ||
114 | // First, shift the high bits down. | |
115 | ushr v2.2d, v1.2d, #63 // the b_i for t | |
116 | ushr v3.2d, v1.2d, #62 // the b_i for t^2 | |
117 | ushr v4.2d, v1.2d, #57 // the b_i for t^7 | |
118 | eor v2.16b, v2.16b, v3.16b // add them all together | |
119 | eor v2.16b, v2.16b, v4.16b | |
120 | vshr128 v3, v2, 64 | |
121 | vshl128 v4, v2, 64 | |
122 | eor v1.16b, v1.16b, v3.16b // contribution into high half | |
123 | eor v0.16b, v0.16b, v4.16b // and low half | |
124 | ||
125 | // And then shift the low bits up. | |
126 | shl v2.2d, v1.2d, #1 | |
127 | shl v3.2d, v1.2d, #2 | |
128 | shl v4.2d, v1.2d, #7 | |
129 | eor v1.16b, v1.16b, v2.16b // unit and t contribs | |
130 | eor v3.16b, v3.16b, v4.16b // t^2 and t^7 contribs | |
131 | eor v0.16b, v0.16b, v1.16b // mix everything together | |
132 | eor v0.16b, v0.16b, v3.16b // ... and we're done | |
133 | .endm | |
134 | ||
135 | .macro mul64 | |
136 | // Enter with u and v in the low halves of v0 and v1, respectively; | |
137 | // leave with z = u v in x2. Clobbers x2--x4. | |
138 | ||
139 | // The multiplication is thankfully easy. | |
140 | // v0 = // (u; ?) | |
141 | // v1 = // (v; ?) | |
142 | pmull v0.1q, v0.1d, v1.1d // u v | |
143 | ||
144 | // Now we must reduce. This is essentially the same as the 128-bit | |
145 | // case above, but mostly simpler because everything is smaller. The | |
146 | // polynomial this time is p(t) = t^64 + t^4 + t^3 + t + 1. | |
147 | ||
148 | // Before we get stuck in, transfer the product to general-purpose | |
149 | // registers. | |
150 | mov x3, v0.d[1] | |
151 | mov x2, v0.d[0] | |
152 | ||
153 | // First, shift the high bits down. | |
154 | eor x4, x3, x3, lsr #1 // pre-mix t^3 and t^4 | |
155 | eor x3, x3, x3, lsr #63 // mix in t contribution | |
156 | eor x3, x3, x4, lsr #60 // shift and mix in t^3 and t^4 | |
157 | ||
158 | // And then shift the low bits up. | |
159 | eor x3, x3, x3, lsl #1 // mix unit and t; pre-mix t^3, t^4 | |
160 | eor x2, x2, x3 // fold them in | |
161 | eor x2, x2, x3, lsl #3 // and t^3 and t^4 | |
162 | .endm | |
163 | ||
164 | .macro mul96 | |
165 | // Enter with u in the least-significant 96 bits of v0, with zero in | |
166 | // the upper 32 bits, and with the least-significant 64 bits of v in | |
167 | // both halves of v1, and the upper 32 bits of v in the low 32 bits | |
168 | // of each half of v2, with zero in the upper 32 bits; and with zero | |
169 | // in v31. Yes, that's a bit hairy. Leave with the product u v in | |
170 | // the low 96 bits of v0, and /junk/ in the high 32 bits. Clobbers | |
171 | // v1--v6. | |
172 | ||
173 | // This is an inconvenient size. There's nothing for it but to do | |
174 | // four multiplications, as if for the 128-bit case. It's possible | |
175 | // that there's cruft in the top 32 bits of the input registers, so | |
176 | // shift both of them up by four bytes before we start. This will | |
177 | // mean that the high 64 bits of the result (from GCM's viewpoint) | |
178 | // will be zero. | |
179 | // v0 = // (u_0 + u_1 t^32; u_2) | |
180 | // v1 = // (v_0 + v_1 t^32; v_0 + v_1 t^32) | |
181 | // v2 = // (v_2; v_2) | |
182 | pmull2 v5.1q, v0.2d, v1.2d // u_2 (v_0 + v_1 t^32) t^32 = e_0 | |
183 | pmull v4.1q, v0.1d, v2.1d // v_2 (u_0 + u_1 t^32) t^32 = e_1 | |
184 | pmull2 v6.1q, v0.2d, v2.2d // u_2 v_2 = d = (d; 0) | |
185 | pmull v3.1q, v0.1d, v1.1d // u_0 v_0 + (u_0 v_1 + u_1 v_0) t^32 | |
186 | // + u_1 v_1 t^64 = f | |
187 | ||
188 | // Extract the high and low halves of the 192-bit result. The answer | |
189 | // we want is d t^128 + e t^64 + f, where e = e_0 + e_1. The low 96 | |
190 | // bits of the answer will end up in v0, with junk in the top 32 | |
191 | // bits; the high 96 bits will end up in v1, which must have zero in | |
192 | // its top 32 bits. | |
193 | // | |
194 | // Here, bot(x) is the low 96 bits of a 192-bit quantity x, arranged | |
195 | // in the low 96 bits of a SIMD register, with junk in the top 32 | |
196 | // bits; and top(x) is the high 96 bits, also arranged in the low 96 | |
197 | // bits of a register, with /zero/ in the top 32 bits. | |
198 | eor v4.16b, v4.16b, v5.16b // e_0 + e_1 = e | |
199 | vshl128 v6, v6, 32 // top(d t^128) | |
200 | vshr128 v5, v4, 32 // top(e t^64) | |
201 | vshl128 v4, v4, 64 // bot(e t^64) | |
202 | vshr128 v1, v3, 96 // top(f) | |
203 | eor v6.16b, v6.16b, v5.16b // top(d t^128 + e t^64) | |
204 | eor v0.16b, v3.16b, v4.16b // bot([d t^128] + e t^64 + f) | |
205 | eor v1.16b, v1.16b, v6.16b // top(e t^64 + d t^128 + f) | |
206 | ||
207 | // Finally, the reduction. This is essentially the same as the | |
208 | // 128-bit case, except that the polynomial is p(t) = t^96 + t^10 + | |
209 | // t^9 + t^6 + 1. The degrees are larger but not enough to cause | |
210 | // trouble for the general approach. Unfortunately, we have to do | |
211 | // this in 32-bit pieces rather than 64. | |
212 | ||
213 | // First, shift the high bits down. | |
214 | ushr v2.4s, v1.4s, #26 // the b_i for t^6 | |
215 | ushr v3.4s, v1.4s, #23 // the b_i for t^9 | |
216 | ushr v4.4s, v1.4s, #22 // the b_i for t^10 | |
217 | eor v2.16b, v2.16b, v3.16b // add them all together | |
218 | eor v2.16b, v2.16b, v4.16b | |
219 | vshr128 v3, v2, 64 // contribution for high half | |
220 | vshl128 v2, v2, 32 // contribution for low half | |
221 | eor v1.16b, v1.16b, v3.16b // apply to high half | |
222 | eor v0.16b, v0.16b, v2.16b // and low half | |
223 | ||
224 | // And then shift the low bits up. | |
225 | shl v2.4s, v1.4s, #6 | |
226 | shl v3.4s, v1.4s, #9 | |
227 | shl v4.4s, v1.4s, #10 | |
228 | eor v1.16b, v1.16b, v2.16b // unit and t^6 contribs | |
229 | eor v3.16b, v3.16b, v4.16b // t^9 and t^10 contribs | |
230 | eor v0.16b, v0.16b, v1.16b // mix everything together | |
231 | eor v0.16b, v0.16b, v3.16b // ... and we're done | |
232 | .endm | |
233 | ||
234 | .macro mul192 | |
235 | // Enter with u in v0 and the less-significant half of v1, with v | |
236 | // duplicated across both halves of v2/v3/v4, and with zero in v31. | |
237 | // Leave with the product u v in v0 and the bottom half of v1. | |
238 | // Clobbers v16--v25. | |
239 | ||
240 | // Start multiplying and accumulating pieces of product. | |
241 | // v0 = // (u_0; u_1) | |
242 | // v1 = // (u_2; ?) | |
243 | // v2 = // (v_0; v_0) | |
244 | // v3 = // (v_1; v_1) | |
245 | // v4 = // (v_2; v_2) | |
246 | pmull v16.1q, v0.1d, v2.1d // a = u_0 v_0 | |
247 | ||
248 | pmull v19.1q, v0.1d, v3.1d // u_0 v_1 | |
249 | pmull2 v21.1q, v0.2d, v2.2d // u_1 v_0 | |
250 | ||
251 | pmull v17.1q, v0.1d, v4.1d // u_0 v_2 | |
252 | pmull2 v22.1q, v0.2d, v3.2d // u_1 v_1 | |
253 | pmull v23.1q, v1.1d, v2.1d // u_2 v_0 | |
254 | eor v19.16b, v19.16b, v21.16b // b = u_0 v_1 + u_1 v_0 | |
255 | ||
256 | pmull2 v20.1q, v0.2d, v4.2d // u_1 v_2 | |
257 | pmull v24.1q, v1.1d, v3.1d // u_2 v_1 | |
258 | eor v17.16b, v17.16b, v22.16b // u_0 v_2 + u_1 v_1 | |
259 | ||
260 | pmull v18.1q, v1.1d, v4.1d // e = u_2 v_2 | |
261 | eor v17.16b, v17.16b, v23.16b // c = u_0 v_2 + u_1 v_1 + u_2 v_1 | |
262 | eor v20.16b, v20.16b, v24.16b // d = u_1 v_2 + u_2 v_1 | |
263 | ||
264 | // Piece the product together. | |
265 | // v16 = // (a_0; a_1) | |
266 | // v19 = // (b_0; b_1) | |
267 | // v17 = // (c_0; c_1) | |
268 | // v20 = // (d_0; d_1) | |
269 | // v18 = // (e_0; e_1) | |
270 | vshl128 v21, v19, 64 // (0; b_0) | |
271 | ext v22.16b, v19.16b, v20.16b, #8 // (b_1; d_0) | |
272 | vshr128 v23, v20, 64 // (d_1; 0) | |
273 | eor v16.16b, v16.16b, v21.16b // (x_0; x_1) | |
274 | eor v17.16b, v17.16b, v22.16b // (x_2; x_3) | |
275 | eor v18.16b, v18.16b, v23.16b // (x_2; x_3) | |
276 | ||
277 | // Next, the reduction. Our polynomial this time is p(x) = t^192 + | |
278 | // t^7 + t^2 + t + 1. Yes, the magic numbers are the same as the | |
279 | // 128-bit case. I don't know why. | |
280 | ||
281 | // First, shift the high bits down. | |
282 | // v16 = // (y_0; y_1) | |
283 | // v17 = // (y_2; y_3) | |
284 | // v18 = // (y_4; y_5) | |
285 | mov v19.d[0], v17.d[1] // (y_3; ?) | |
286 | ||
287 | ushr v23.2d, v18.2d, #63 // hi b_i for t | |
288 | ushr d20, d19, #63 // lo b_i for t | |
289 | ushr v24.2d, v18.2d, #62 // hi b_i for t^2 | |
290 | ushr d21, d19, #62 // lo b_i for t^2 | |
291 | ushr v25.2d, v18.2d, #57 // hi b_i for t^7 | |
292 | ushr d22, d19, #57 // lo b_i for t^7 | |
293 | eor v23.16b, v23.16b, v24.16b // mix them all together | |
294 | eor v20.8b, v20.8b, v21.8b | |
295 | eor v23.16b, v23.16b, v25.16b | |
296 | eor v20.8b, v20.8b, v22.8b | |
297 | ||
298 | // Permute the high pieces while we fold in the b_i. | |
299 | eor v17.16b, v17.16b, v23.16b | |
300 | vshl128 v20, v20, 64 | |
301 | mov v19.d[0], v18.d[1] // (y_5; ?) | |
302 | ext v18.16b, v17.16b, v18.16b, #8 // (y_3; y_4) | |
303 | eor v16.16b, v16.16b, v20.16b | |
304 | ||
305 | // And finally shift the low bits up. | |
306 | // v16 = // (y'_0; y'_1) | |
307 | // v17 = // (y'_2; ?) | |
308 | // v18 = // (y'_3; y'_4) | |
309 | // v19 = // (y'_5; ?) | |
310 | shl v20.2d, v18.2d, #1 | |
311 | shl d23, d19, #1 | |
312 | shl v21.2d, v18.2d, #2 | |
313 | shl d24, d19, #2 | |
314 | shl v22.2d, v18.2d, #7 | |
315 | shl d25, d19, #7 | |
316 | eor v18.16b, v18.16b, v20.16b // unit and t contribs | |
317 | eor v19.8b, v19.8b, v23.8b | |
318 | eor v21.16b, v21.16b, v22.16b // t^2 and t^7 contribs | |
319 | eor v24.8b, v24.8b, v25.8b | |
320 | eor v18.16b, v18.16b, v21.16b // all contribs | |
321 | eor v19.8b, v19.8b, v24.8b | |
322 | eor v0.16b, v16.16b, v18.16b // mix them into the low half | |
323 | eor v1.8b, v17.8b, v19.8b | |
324 | .endm | |
325 | ||
326 | .macro mul256 | |
327 | // Enter with u in v0/v1, with v duplicated across both halves of | |
328 | // v2--v5, and with zero in v31. Leave with the product u v in | |
329 | // v0/v1. Clobbers ???. | |
330 | ||
331 | // Now it's starting to look worthwhile to do Karatsuba. Suppose | |
332 | // u = u_0 + u_1 B and v = v_0 + v_1 B. Then | |
333 | // | |
334 | // u v = (u_0 v_0) + (u_0 v_1 + u_1 v_0) B + (u_1 v_1) B^2 | |
335 | // | |
336 | // Name these coefficients of B^i be a, b, and c, respectively, and | |
337 | // let r = u_0 + u_1 and s = v_0 + v_1. Then observe that | |
338 | // | |
339 | // q = r s = (u_0 + u_1) (v_0 + v_1) | |
340 | // = (u_0 v_0) + (u1 v_1) + (u_0 v_1 + u_1 v_0) | |
341 | // = a + d + c | |
342 | // | |
343 | // The first two terms we've already calculated; the last is the | |
344 | // remaining one we want. We'll set B = t^128. We know how to do | |
345 | // 128-bit multiplications already, and Karatsuba is too annoying | |
346 | // there, so there'll be 12 multiplications altogether, rather than | |
347 | // the 16 we'd have if we did this the naïve way. | |
348 | // v0 = // u_0 = (u_00; u_01) | |
349 | // v1 = // u_1 = (u_10; u_11) | |
350 | // v2 = // (v_00; v_00) | |
351 | // v3 = // (v_01; v_01) | |
352 | // v4 = // (v_10; v_10) | |
353 | // v5 = // (v_11; v_11) | |
354 | ||
355 | eor v28.16b, v0.16b, v1.16b // u_* = (u_00 + u_10; u_01 + u_11) | |
356 | eor v29.16b, v2.16b, v4.16b // v_*0 = v_00 + v_10 | |
357 | eor v30.16b, v3.16b, v5.16b // v_*1 = v_01 + v_11 | |
358 | ||
359 | // Start by building the cross product, q = u_* v_*. | |
360 | pmull v24.1q, v28.1d, v30.1d // u_*0 v_*1 | |
361 | pmull2 v25.1q, v28.2d, v29.2d // u_*1 v_*0 | |
362 | pmull v20.1q, v28.1d, v29.1d // u_*0 v_*0 | |
363 | pmull2 v21.1q, v28.2d, v30.2d // u_*1 v_*1 | |
364 | eor v24.16b, v24.16b, v25.16b // u_*0 v_*1 + u_*1 v_*0 | |
365 | vshr128 v25, v24, 64 | |
366 | vshl128 v24, v24, 64 | |
367 | eor v20.16b, v20.16b, v24.16b // q_0 | |
368 | eor v21.16b, v21.16b, v25.16b // q_1 | |
369 | ||
370 | // Next, work on the low half, a = u_0 v_0 | |
371 | pmull v24.1q, v0.1d, v3.1d // u_00 v_01 | |
372 | pmull2 v25.1q, v0.2d, v2.2d // u_01 v_00 | |
373 | pmull v16.1q, v0.1d, v2.1d // u_00 v_00 | |
374 | pmull2 v17.1q, v0.2d, v3.2d // u_01 v_01 | |
375 | eor v24.16b, v24.16b, v25.16b // u_00 v_01 + u_01 v_00 | |
376 | vshr128 v25, v24, 64 | |
377 | vshl128 v24, v24, 64 | |
378 | eor v16.16b, v16.16b, v24.16b // a_0 | |
379 | eor v17.16b, v17.16b, v25.16b // a_1 | |
380 | ||
381 | // Mix the pieces we have so far. | |
382 | eor v20.16b, v20.16b, v16.16b | |
383 | eor v21.16b, v21.16b, v17.16b | |
384 | ||
385 | // Finally, work on the high half, c = u_1 v_1 | |
386 | pmull v24.1q, v1.1d, v5.1d // u_10 v_11 | |
387 | pmull2 v25.1q, v1.2d, v4.2d // u_11 v_10 | |
388 | pmull v18.1q, v1.1d, v4.1d // u_10 v_10 | |
389 | pmull2 v19.1q, v1.2d, v5.2d // u_11 v_11 | |
390 | eor v24.16b, v24.16b, v25.16b // u_10 v_11 + u_11 v_10 | |
391 | vshr128 v25, v24, 64 | |
392 | vshl128 v24, v24, 64 | |
393 | eor v18.16b, v18.16b, v24.16b // c_0 | |
394 | eor v19.16b, v19.16b, v25.16b // c_1 | |
395 | ||
396 | // Finish mixing the product together. | |
397 | eor v20.16b, v20.16b, v18.16b | |
398 | eor v21.16b, v21.16b, v19.16b | |
399 | eor v17.16b, v17.16b, v20.16b | |
400 | eor v18.16b, v18.16b, v21.16b | |
401 | ||
402 | // Now we must reduce. This is essentially the same as the 192-bit | |
403 | // case above, but more complicated because everything is bigger. | |
404 | // The polynomial this time is p(t) = t^256 + t^10 + t^5 + t^2 + 1. | |
405 | // v16 = // (y_0; y_1) | |
406 | // v17 = // (y_2; y_3) | |
407 | // v18 = // (y_4; y_5) | |
408 | // v19 = // (y_6; y_7) | |
409 | ushr v24.2d, v18.2d, #62 // (y_4; y_5) b_i for t^2 | |
410 | ushr v25.2d, v19.2d, #62 // (y_6; y_7) b_i for t^2 | |
411 | ushr v26.2d, v18.2d, #59 // (y_4; y_5) b_i for t^5 | |
412 | ushr v27.2d, v19.2d, #59 // (y_6; y_7) b_i for t^5 | |
413 | ushr v28.2d, v18.2d, #54 // (y_4; y_5) b_i for t^10 | |
414 | ushr v29.2d, v19.2d, #54 // (y_6; y_7) b_i for t^10 | |
415 | eor v24.16b, v24.16b, v26.16b // mix the contributions together | |
416 | eor v25.16b, v25.16b, v27.16b | |
417 | eor v24.16b, v24.16b, v28.16b | |
418 | eor v25.16b, v25.16b, v29.16b | |
419 | vshr128 v26, v25, 64 // slide contribs into position | |
420 | ext v25.16b, v24.16b, v25.16b, #8 | |
421 | vshl128 v24, v24, 64 | |
422 | eor v18.16b, v18.16b, v26.16b | |
423 | eor v17.16b, v17.16b, v25.16b | |
424 | eor v16.16b, v16.16b, v24.16b | |
425 | ||
426 | // And then shift the low bits up. | |
427 | // v16 = // (y'_0; y'_1) | |
428 | // v17 = // (y'_2; y'_3) | |
429 | // v18 = // (y'_4; y'_5) | |
430 | // v19 = // (y'_6; y'_7) | |
431 | shl v24.2d, v18.2d, #2 // (y'_4; y_5) a_i for t^2 | |
432 | shl v25.2d, v19.2d, #2 // (y_6; y_7) a_i for t^2 | |
433 | shl v26.2d, v18.2d, #5 // (y'_4; y_5) a_i for t^5 | |
434 | shl v27.2d, v19.2d, #5 // (y_6; y_7) a_i for t^5 | |
435 | shl v28.2d, v18.2d, #10 // (y'_4; y_5) a_i for t^10 | |
436 | shl v29.2d, v19.2d, #10 // (y_6; y_7) a_i for t^10 | |
437 | eor v18.16b, v18.16b, v24.16b // mix the contributions together | |
438 | eor v19.16b, v19.16b, v25.16b | |
439 | eor v26.16b, v26.16b, v28.16b | |
440 | eor v27.16b, v27.16b, v29.16b | |
441 | eor v18.16b, v18.16b, v26.16b | |
442 | eor v19.16b, v19.16b, v27.16b | |
443 | eor v0.16b, v16.16b, v18.16b | |
444 | eor v1.16b, v17.16b, v19.16b | |
445 | .endm | |
446 | ||
447 | ///-------------------------------------------------------------------------- | |
448 | /// Main code. | |
449 | ||
450 | // There are a number of representations of field elements in this code and | |
451 | // it can be confusing. | |
452 | // | |
453 | // * The `external format' consists of a sequence of contiguous bytes in | |
454 | // memory called a `block'. The GCM spec explains how to interpret this | |
455 | // block as an element of a finite field. As discussed extensively, this | |
456 | // representation is very annoying for a number of reasons. On the other | |
457 | // hand, this code never actually deals with it directly. | |
458 | // | |
459 | // * The `register format' consists of one or more SIMD registers, | |
460 | // depending on the block size. The bits in each byte are reversed, | |
461 | // compared to the external format, which makes the polynomials | |
462 | // completely vanilla, unlike all of the other GCM implementations. | |
463 | // | |
464 | // * The `table format' is just like the `register format', only the two | |
465 | // halves of 128-bit SIMD register are the same, so we need twice as many | |
466 | // registers. | |
467 | // | |
468 | // * The `words' format consists of a sequence of bytes, as in the | |
469 | // `external format', but, according to the blockcipher in use, the bytes | |
470 | // within each 32-bit word may be reversed (`big-endian') or not | |
471 | // (`little-endian'). Accordingly, there are separate entry points for | |
472 | // each variant, identified with `b' or `l'. | |
473 | ||
474 | FUNC(gcm_mulk_128b_arm64_pmull) | |
475 | // On entry, x0 points to a 128-bit field element A in big-endian | |
476 | // words format; x1 points to a field-element K in table format. On | |
477 | // exit, A is updated with the product A K. | |
478 | ||
479 | ldr q0, [x0] | |
480 | ldp q1, q2, [x1] | |
481 | rev32 v0.16b, v0.16b | |
482 | vzero | |
483 | rbit v0.16b, v0.16b | |
484 | mul128 | |
485 | rbit v0.16b, v0.16b | |
486 | rev32 v0.16b, v0.16b | |
487 | str q0, [x0] | |
488 | ret | |
489 | ENDFUNC | |
490 | ||
491 | FUNC(gcm_mulk_128l_arm64_pmull) | |
492 | // On entry, x0 points to a 128-bit field element A in little-endian | |
493 | // words format; x1 points to a field-element K in table format. On | |
494 | // exit, A is updated with the product A K. | |
495 | ||
496 | ldr q0, [x0] | |
497 | ldp q1, q2, [x1] | |
498 | vzero | |
499 | rbit v0.16b, v0.16b | |
500 | mul128 | |
501 | rbit v0.16b, v0.16b | |
502 | str q0, [x0] | |
503 | ret | |
504 | ENDFUNC | |
505 | ||
506 | FUNC(gcm_mulk_64b_arm64_pmull) | |
507 | // On entry, x0 points to a 64-bit field element A in big-endian | |
508 | // words format; x1 points to a field-element K in table format. On | |
509 | // exit, A is updated with the product A K. | |
510 | ||
511 | ldr d0, [x0] | |
512 | ldr q1, [x1] | |
513 | rev32 v0.8b, v0.8b | |
514 | rbit v0.8b, v0.8b | |
515 | mul64 | |
516 | rbit x2, x2 | |
517 | ror x2, x2, #32 | |
518 | str x2, [x0] | |
519 | ret | |
520 | ENDFUNC | |
521 | ||
522 | FUNC(gcm_mulk_64l_arm64_pmull) | |
523 | // On entry, x0 points to a 64-bit field element A in little-endian | |
524 | // words format; x1 points to a field-element K in table format. On | |
525 | // exit, A is updated with the product A K. | |
526 | ||
527 | ldr d0, [x0] | |
528 | ldr q1, [x1] | |
529 | rbit v0.8b, v0.8b | |
530 | mul64 | |
531 | rbit x2, x2 | |
532 | rev x2, x2 | |
533 | str x2, [x0] | |
534 | ret | |
535 | ENDFUNC | |
536 | ||
537 | FUNC(gcm_mulk_96b_arm64_pmull) | |
538 | // On entry, x0 points to a 96-bit field element A in big-endian | |
539 | // words format; x1 points to a field-element K in table format. On | |
540 | // exit, A is updated with the product A K. | |
541 | ||
542 | ldr w2, [x0, #8] | |
543 | ldr d0, [x0, #0] | |
544 | mov v0.d[1], x2 | |
545 | ldp q1, q2, [x1] | |
546 | rev32 v0.16b, v0.16b | |
547 | vzero | |
548 | rbit v0.16b, v0.16b | |
549 | mul96 | |
550 | rbit v0.16b, v0.16b | |
551 | rev32 v0.16b, v0.16b | |
552 | mov w2, v0.s[2] | |
553 | str d0, [x0, #0] | |
554 | str w2, [x0, #8] | |
555 | ret | |
556 | ENDFUNC | |
557 | ||
558 | FUNC(gcm_mulk_96l_arm64_pmull) | |
559 | // On entry, x0 points to a 96-bit field element A in little-endian | |
560 | // words format; x1 points to a field-element K in table format. On | |
561 | // exit, A is updated with the product A K. | |
562 | ||
563 | ldr d0, [x0, #0] | |
564 | ldr w2, [x0, #8] | |
565 | mov v0.d[1], x2 | |
566 | ldp q1, q2, [x1] | |
567 | rbit v0.16b, v0.16b | |
568 | vzero | |
569 | mul96 | |
570 | rbit v0.16b, v0.16b | |
571 | mov w2, v0.s[2] | |
572 | str d0, [x0, #0] | |
573 | str w2, [x0, #8] | |
574 | ret | |
575 | ENDFUNC | |
576 | ||
577 | FUNC(gcm_mulk_192b_arm64_pmull) | |
578 | // On entry, x0 points to a 192-bit field element A in big-endian | |
579 | // words format; x1 points to a field-element K in table format. On | |
580 | // exit, A is updated with the product A K. | |
581 | ||
582 | ldr q0, [x0, #0] | |
583 | ldr d1, [x0, #16] | |
584 | ldp q2, q3, [x1, #0] | |
585 | ldr q4, [x1, #32] | |
586 | rev32 v0.16b, v0.16b | |
587 | rev32 v1.8b, v1.8b | |
588 | rbit v0.16b, v0.16b | |
589 | rbit v1.8b, v1.8b | |
590 | vzero | |
591 | mul192 | |
592 | rev32 v0.16b, v0.16b | |
593 | rev32 v1.8b, v1.8b | |
594 | rbit v0.16b, v0.16b | |
595 | rbit v1.8b, v1.8b | |
596 | str q0, [x0, #0] | |
597 | str d1, [x0, #16] | |
598 | ret | |
599 | ENDFUNC | |
600 | ||
601 | FUNC(gcm_mulk_192l_arm64_pmull) | |
602 | // On entry, x0 points to a 192-bit field element A in little-endian | |
603 | // words format; x1 points to a field-element K in table format. On | |
604 | // exit, A is updated with the product A K. | |
605 | ||
606 | ldr q0, [x0, #0] | |
607 | ldr d1, [x0, #16] | |
608 | ldp q2, q3, [x1, #0] | |
609 | ldr q4, [x1, #32] | |
610 | rbit v0.16b, v0.16b | |
611 | rbit v1.8b, v1.8b | |
612 | vzero | |
613 | mul192 | |
614 | rbit v0.16b, v0.16b | |
615 | rbit v1.8b, v1.8b | |
616 | str q0, [x0, #0] | |
617 | str d1, [x0, #16] | |
618 | ret | |
619 | ENDFUNC | |
620 | ||
621 | FUNC(gcm_mulk_256b_arm64_pmull) | |
622 | // On entry, x0 points to a 256-bit field element A in big-endian | |
623 | // words format; x1 points to a field-element K in table format. On | |
624 | // exit, A is updated with the product A K. | |
625 | ||
626 | ldp q0, q1, [x0] | |
627 | ldp q2, q3, [x1, #0] | |
628 | ldp q4, q5, [x1, #32] | |
629 | rev32 v0.16b, v0.16b | |
630 | rev32 v1.16b, v1.16b | |
631 | rbit v0.16b, v0.16b | |
632 | rbit v1.16b, v1.16b | |
633 | vzero | |
634 | mul256 | |
635 | rev32 v0.16b, v0.16b | |
636 | rev32 v1.16b, v1.16b | |
637 | rbit v0.16b, v0.16b | |
638 | rbit v1.16b, v1.16b | |
639 | stp q0, q1, [x0] | |
640 | ret | |
641 | ENDFUNC | |
642 | ||
643 | FUNC(gcm_mulk_256l_arm64_pmull) | |
644 | // On entry, x0 points to a 256-bit field element A in little-endian | |
645 | // words format; x1 points to a field-element K in table format. On | |
646 | // exit, A is updated with the product A K. | |
647 | ||
648 | ldp q0, q1, [x0] | |
649 | ldp q2, q3, [x1, #0] | |
650 | ldp q4, q5, [x1, #32] | |
651 | rbit v0.16b, v0.16b | |
652 | rbit v1.16b, v1.16b | |
653 | vzero | |
654 | mul256 | |
655 | rbit v0.16b, v0.16b | |
656 | rbit v1.16b, v1.16b | |
657 | stp q0, q1, [x0] | |
658 | ret | |
659 | ENDFUNC | |
660 | ||
661 | ///----- That's all, folks -------------------------------------------------- |