/// pieces are placed into 32-bit cells, and arranged as two 128-bit SSE
/// operands, as follows.
///
-/// Offset 0 4 8 12
-/// 0 v'_0 v'_1 v''_0 v''_1
-/// 16 v'_2 v'_3 v''_2 v''_3
+/// Offset 12 8 4 0
+/// 0 v''_1 v''_0 v'_1 v'_0
+/// 16 v''_3 v''_2 v'_3 v'_2
///
/// A `pmuludq' instruction ignores the odd positions in its operands; thus,
/// it will act on (say) v'_0 and v''_0 in a single instruction. Shifting
.macro mulcore r, s, d0, d1=nil, d2=nil, d3=nil
// Load a word r_i from R, multiply by the expanded operand [S], and
// leave the pieces of the product in registers D0, D1, D2, D3.
- movd \d0, \r // (r_i, 0; 0, 0)
+ movd \d0, \r // (0, 0; 0, r_i)
.ifnes "\d1", "nil"
- movdqa \d1, [\s] // (s'_0, s'_1; s''_0, s''_1)
+ movdqa \d1, [\s] // (s''_1, s''_0; s'_1, s'_0)
.endif
.ifnes "\d3", "nil"
- movdqa \d3, [\s + 16] // (s'_2, s'_3; s''_2, s''_3)
+ movdqa \d3, [\s + 16] // (s''_3, s''_2; s'_3, s'_2)
.endif
- pshufd \d0, \d0, SHUF(0, 3, 0, 3) // (r_i, ?; r_i, ?)
+ pshufd \d0, \d0, SHUF(3, 0, 3, 0) // (?, r_i; ?, r_i)
.ifnes "\d1", "nil"
- psrldq \d1, 4 // (s'_1, s''_0; s''_1, 0)
+ psrldq \d1, 4 // (0, s''_1; s''_0, s'_1)
.endif
.ifnes "\d2", "nil"
.ifnes "\d3", "nil"
movdqa \d2, \d3 // another copy of (s'_2, s'_3; ...)
.else
- movdqa \d2, \d0 // another copy of (r_i, ?; r_i, ?)
+ movdqa \d2, \d0 // another copy of (?, r_i; ?, r_i)
.endif
.endif
.ifnes "\d3", "nil"
- psrldq \d3, 4 // (s'_3, s''_2; s''_3, 0)
+ psrldq \d3, 4 // (0, s''_3; s''_2, s'_3)
.endif
.ifnes "\d1", "nil"
- pmuludq \d1, \d0 // (r_i s'_1; r_i s''_1)
+ pmuludq \d1, \d0 // (r_i s''_1; r_i s'_1)
.endif
.ifnes "\d3", "nil"
- pmuludq \d3, \d0 // (r_i s'_3; r_i s''_3)
+ pmuludq \d3, \d0 // (r_i s''_3; r_i s'_3)
.endif
.ifnes "\d2", "nil"
.ifnes "\d3", "nil"
- pmuludq \d2, \d0 // (r_i s'_2; r_i s''_2)
+ pmuludq \d2, \d0 // (r_i s''_2; r_i s'_2)
.else
pmuludq \d2, [\s + 16]
.endif
.endif
- pmuludq \d0, [\s] // (r_i s'_0; r_i s''_0)
+ pmuludq \d0, [\s] // (r_i s''_0; r_i s'_0)
.endm
.macro accum c0, c1=nil, c2=nil, c3=nil
// carry registers. On completion, XMM3 is clobbered. If CC is
// `nil', then the contribution which would have been added to it is
// left in C.
- pshufd xmm3, \c, SHUF(3, 3, 3, 2) // (?, ?; ?, t = c'' mod B)
- psrldq xmm3, 12 // (t, 0; 0, 0) = (t, 0)
- pslldq xmm3, 2 // (t b; 0)
- paddq \c, xmm3 // (c' + t b; c'')
+ pshufd xmm3, \c, SHUF(2, 3, 3, 3) // (t = c'' mod B, ?; ?, ?)
+ psrldq xmm3, 12 // (0, 0; 0, t) = (0; t)
+ pslldq xmm3, 2 // (0; t b)
+ paddq \c, xmm3 // (c''; c' + t b)
movd \d, \c
psrlq \c, 32 // floor(c/B)
.ifnes "\cc", "nil"
// of the value represented in C are written to D, and the remaining
// bits are left at the bottom of T.
movdqa \t, \c
- psllq \t, 16 // (?; c'' b)
- pslldq \c, 8 // (0; c')
- paddq \t, \c // (?; c' + c'' b)
- psrldq \t, 8 // (c' + c'' b; 0) = (c; 0)
+ psllq \t, 16 // (c'' b; ?)
+ pslldq \c, 8 // (c'; 0)
+ paddq \t, \c // (c' + c'' b; ?)
+ psrldq \t, 8 // (0; c' + c'' b) = (0; c)
movd \d, \t
psrldq \t, 4 // (floor(c/B); 0)
.endm
// On entry, A and C hold packed 128-bit values, and Z is zero. On
// exit, A:B and C:D together hold the same values in expanded
// form. If C is `nil', then only expand A to A:B.
- movdqa \b, \a // (a_0, a_1; a_2, a_3)
+ movdqa \b, \a // (a_3, a_2; a_1, a_0)
.ifnes "\c", "nil"
- movdqa \d, \c // (c_0, c_1; c_2, c_3)
+ movdqa \d, \c // (c_3, c_2; c_1, c_0)
.endif
- punpcklwd \a, \z // (a'_0, a''_0; a'_1, a''_1)
- punpckhwd \b, \z // (a'_2, a''_2; a'_3, a''_3)
+ punpcklwd \a, \z // (a''_1, a'_1; a''_0, a'_0)
+ punpckhwd \b, \z // (a''_3, a'_3; a''_2, a'_2)
.ifnes "\c", "nil"
- punpcklwd \c, \z // (c'_0, c''_0; c'_1, c''_1)
- punpckhwd \d, \z // (c'_2, c''_2; c'_3, c''_3)
+ punpcklwd \c, \z // (c''_1, c'_1; c''_0, c'_0)
+ punpckhwd \d, \z // (c''_3, c'_3; c''_2, c'_2)
.endif
- pshufd \a, \a, SHUF(0, 2, 1, 3) // (a'_0, a'_1; a''_0, a''_1)
- pshufd \b, \b, SHUF(0, 2, 1, 3) // (a'_2, a'_3; a''_2, a''_3)
+ pshufd \a, \a, SHUF(3, 1, 2, 0) // (a''_1, a''_0; a'_1, a'_0)
+ pshufd \b, \b, SHUF(3, 1, 2, 0) // (a''_3, a''_2; a'_3, a'_2)
.ifnes "\c", "nil"
- pshufd \c, \c, SHUF(0, 2, 1, 3) // (c'_0, c'_1; c''_0, c''_1)
- pshufd \d, \d, SHUF(0, 2, 1, 3) // (c'_2, c'_3; c''_2, c''_3)
+ pshufd \c, \c, SHUF(3, 1, 2, 0) // (c''_1, c''_0; c'_1, c'_0)
+ pshufd \d, \d, SHUF(3, 1, 2, 0) // (c''_3, c''_2; c'_3, c'_2)
.endif
.endm
// we can do that, we must gather them together.
movdqa \t, \c0
movdqa \u, \c1
- punpcklqdq \t, \c2 // (y'_0; y'_2)
- punpckhqdq \c0, \c2 // (y''_0; y''_2)
- punpcklqdq \u, \c3 // (y'_1; y'_3)
- punpckhqdq \c1, \c3 // (y''_1; y''_3)
+ punpcklqdq \t, \c2 // (y'_2; y'_0)
+ punpckhqdq \c0, \c2 // (y''_2; y''_0)
+ punpcklqdq \u, \c3 // (y'_3; y'_1)
+ punpckhqdq \c1, \c3 // (y''_3; y''_1)
// Now split the double-prime pieces. The high (up to) 48 bits will
// go up; the low 16 bits go down.
movdqa \c3, \c1
psllq \c2, 48
psllq \c3, 48
- psrlq \c0, 16 // high parts of (y''_0; y''_2)
- psrlq \c1, 16 // high parts of (y''_1; y''_3)
- psrlq \c2, 32 // low parts of (y''_0; y''_2)
- psrlq \c3, 32 // low parts of (y''_1; y''_3)
+ psrlq \c0, 16 // high parts of (y''_2; y''_0)
+ psrlq \c1, 16 // high parts of (y''_3; y''_1)
+ psrlq \c2, 32 // low parts of (y''_2; y''_0)
+ psrlq \c3, 32 // low parts of (y''_3; y''_1)
.ifnes "\hi", "nil"
movdqa \hi, \c1
.endif
- pslldq \c1, 8 // high part of (0; y''_1)
+ pslldq \c1, 8 // high part of (y''_1; 0)
paddq \t, \c2 // propagate down
paddq \u, \c3
- paddq \t, \c1 // and up: (y_0; y_2)
- paddq \u, \c0 // (y_1; y_3)
+ paddq \t, \c1 // and up: (y_2; y_0)
+ paddq \u, \c0 // (y_3; y_1)
.ifnes "\hi", "nil"
- psrldq \hi, 8 // high part of (y''_3; 0)
+ psrldq \hi, 8 // high part of (0; y''_3)
.endif
// Finally extract the answer. This complicated dance is better than
// storing to memory and loading, because the piecemeal stores
// inhibit store forwarding.
- movdqa \c3, \t // (y_0; ?)
- movdqa \lo, \t // (y^*_0, ?; ?, ?)
- psrldq \t, 8 // (y_2; 0)
+ movdqa \c3, \t // (?; y_0)
+ movdqa \lo, \t // (?, ?; ?, y^*_0)
+ psrldq \t, 8 // (0; y_2)
psrlq \c3, 32 // (floor(y_0/B); ?)
paddq \c3, \u // (y_1 + floor(y_0/B); ?)
- movdqa \c1, \c3 // (y^*_1, ?; ?, ?)
- psrldq \u, 8 // (y_3; 0)
+ movdqa \c1, \c3 // (?, ?; ?, y^*_1)
+ psrldq \u, 8 // (0; y_3)
psrlq \c3, 32 // (floor((y_1 B + y_0)/B^2; ?)
paddq \c3, \t // (y_2 + floor((y_1 B + y_0)/B^2; ?)
- punpckldq \lo, \c3 // (y^*_0, y^*_2; ?, ?)
+ punpckldq \lo, \c3 // (?, ?; y^*_2, y^*_0)
psrlq \c3, 32 // (floor((y_2 B^2 + y_1 B + y_0)/B^3; ?)
paddq \c3, \u // (y_3 + floor((y_2 B^2 + y_1 B + y_0)/B^3; ?)
.ifnes "\hi", "nil"
movdqa \t, \c3
pxor \u, \u
.endif
- punpckldq \c1, \c3 // (y^*_1, y^*_3; ?, ?)
+ punpckldq \c1, \c3 // (?, ?; y^*_3, y^*_1)
.ifnes "\hi", "nil"
psrlq \t, 32 // very high bits of y
paddq \hi, \t
// On exit, the carry registers, including XMM7, are updated to hold
// C + A; XMM0, XMM1, XMM2, and XMM3 are clobbered. The other
// registers are preserved.
- movd xmm0, [edi + 0] // (a_0; 0)
- movd xmm1, [edi + 4] // (a_1; 0)
- movd xmm2, [edi + 8] // (a_2; 0)
- movd xmm7, [edi + 12] // (a_3; 0)
-
- paddq xmm4, xmm0 // (c'_0 + a_0; c''_0)
- paddq xmm5, xmm1 // (c'_1 + a_1; c''_1)
- paddq xmm6, xmm2 // (c'_2 + a_2; c''_2 + a_3 b)
+ movd xmm0, [edi + 0] // (0; a_0)
+ movd xmm1, [edi + 4] // (0; a_1)
+ movd xmm2, [edi + 8] // (0; a_2)
+ movd xmm7, [edi + 12] // (0; a_3)
+
+ paddq xmm4, xmm0 // (c''_0; c'_0 + a_0)
+ paddq xmm5, xmm1 // (c''_1; c'_1 + a_1)
+ paddq xmm6, xmm2 // (c''_2 + a_3 b; c'_2 + a_2)
.endm
///--------------------------------------------------------------------------
.macro testldcarry c
mov ecx, \c // -> c
- movdqu xmm4, [ecx + 0] // (c'_0; c''_0)
- movdqu xmm5, [ecx + 16] // (c'_1; c''_1)
- movdqu xmm6, [ecx + 32] // (c'_2; c''_2)
+ movdqu xmm4, [ecx + 0] // (c''_0; c'_0)
+ movdqu xmm5, [ecx + 16] // (c''_1; c'_1)
+ movdqu xmm6, [ecx + 32] // (c''_2; c'_2)
.endm
.macro testexpand v=nil, y=nil
mov edi, [BP + 28]
movdqa xmm0, [SP + 64]
movdqa xmm1, [SP + 80]
- pshufd xmm0, xmm0, SHUF(0, 2, 1, 3)
- pshufd xmm1, xmm1, SHUF(0, 2, 1, 3)
+ pshufd xmm0, xmm0, SHUF(3, 1, 2, 0)
+ pshufd xmm1, xmm1, SHUF(3, 1, 2, 0)
movdqu [edi], xmm0
movdqu [edi + 16], xmm1
testcarryout [BP + 24]
mov edi, [BP + 28]
movdqa xmm0, [SP + 64]
movdqa xmm1, [SP + 80]
- pshufd xmm0, xmm0, SHUF(0, 2, 1, 3)
- pshufd xmm1, xmm1, SHUF(0, 2, 1, 3)
+ pshufd xmm0, xmm0, SHUF(3, 1, 2, 0)
+ pshufd xmm1, xmm1, SHUF(3, 1, 2, 0)
movdqu [edi], xmm0
movdqu [edi + 16], xmm1
testcarryout [BP + 24]
mov edi, [BP + 28]
movdqa xmm0, [SP + 64]
movdqa xmm1, [SP + 80]
- pshufd xmm0, xmm0, SHUF(0, 2, 1, 3)
- pshufd xmm1, xmm1, SHUF(0, 2, 1, 3)
+ pshufd xmm0, xmm0, SHUF(3, 1, 2, 0)
+ pshufd xmm1, xmm1, SHUF(3, 1, 2, 0)
movdqu [edi], xmm0
movdqu [edi + 16], xmm1
testcarryout [BP + 24]
~mdw / catacomb / blobdiff