/// construct more general variable-length multipliers.
///
/// The basic trick is the same throughout. In an operand-scanning
-/// multiplication, the inner multiplication loop multiplies a
-/// multiple-precision operand by a single precision factor, and adds the
-/// result, appropriately shifted, to the result. A `finely integrated
-/// operand scanning' implementation of Montgomery multiplication also adds
-/// the product of a single-precision `Montgomery factor' and the modulus,
+/// multiplication, the inner multiplication loop multiplies a multiple-
+/// precision operand by a single precision factor, and adds the result,
+/// appropriately shifted, to the result. A `finely integrated operand
+/// scanning' implementation of Montgomery multiplication also adds the
+/// product of a single-precision `Montgomery factor' and the modulus,
/// calculated in the same pass. The more common `coarsely integrated
/// operand scanning' alternates main multiplication and Montgomery passes,
/// which requires additional carry propagation.
/// many products together before we must deal with carrying; it also allows
/// for some calculations to be performed on the above expanded form.
///
-/// ...
+/// We maintain four `carry' registers XMM12--XMM15 accumulating intermediate
+/// results. The registers' precise roles rotate during the computation; we
+/// name them `c0', `c1', `c2', and `c3'. Each carry register holds two
+/// 64-bit halves: the register c0, for example, holds c'_0 (low half) and
+/// c''_0 (high half), and represents the value c_0 = c'_0 + c''_0 b; the
+/// carry registers collectively represent the value c_0 + c_1 B + c_2 B^2 +
+/// c_3 B^3. The `pmuluqdq' instruction acting on a scalar operand
+/// (broadcast across all lanes of its vector) and an operand in the expanded
+/// form above produces a result which can be added directly to the
+/// appropriate carry register. Following a pass of four multiplications, we
+/// perform some limited carry propagation: let t = c''_0 mod B, and let d =
+/// c'_0 + t b; then we output z = d mod B, add (floor(d/B), floor(c''_0/B))
+/// to c1, and cycle the carry registers around, so that c1 becomes c0, and
+/// the old (implicitly) zeroed c0 becomes c3.
///
-/// We maintain four `carry' registers accumulating intermediate results.
-/// The registers' precise roles rotate during the computation; we name them
-/// `c0', `c1', `c2', and `c3'. Each carry register holds two 64-bit halves:
-/// the register c0, for example, holds c'_0 (low half) and c''_0 (high
-/// half), and represents the value c_0 = c'_0 + c''_0 b; the carry registers
-/// collectively represent the value c_0 + c_1 B + c_2 B^2 + c_3 B^3. The
-/// `pmuluqdq' instruction acting on a scalar operand (broadcast across all
-/// lanes of its vector) and an operand in the expanded form above produces a
-/// result which can be added directly to the appropriate carry register.
-/// Following a pass of four multiplications, we perform some limited carry
-/// propagation: let t = c''_0 mod B, and let d = c'_0 + t b; then we output
-/// z = d mod B, add (floor(d/B), floor(c''_0/B)) to c1, and cycle the carry
-/// registers around, so that c1 becomes c0, and the old c0 is (implicitly)
-/// zeroed becomes c3.
+/// On 64-bit AMD64, we have a reasonable number of registers: the expanded
+/// operands are kept in registers. The packed operands are read from memory
+/// into working registers XMM4 and XMM5; XMM0--XMM3 are used for the actual
+/// multiplications; and XMM6 and XMM7 are used for combining the results.
+/// The following conventional argument names and locations are used
+/// throughout.
+///
+/// Arg Format Location Notes
+///
+/// U packed [RAX]
+/// X packed [RBX] In Montgomery multiplication, X = N
+/// V expanded XMM8/XMM9
+/// Y expanded XMM10/XMM11 In Montgomery multiplication, Y = (A + U V) M
+/// M expanded (see below) Montgomery factor, M = -N^{-1} (mod B^4)
+/// N Modulus, for Montgomery multiplication
+/// A packed [RDI] Destination/accumulator
+/// C carry XMM12--XMM15
+///
+/// The calculation is some variant of
+///
+/// A' + C' B^4 <- U V + X Y + A + C
+///
+/// The low-level functions fit into a fairly traditional (finely-integrated)
+/// operand scanning loop over operand pairs (U, X) (indexed by j) and (V, Y)
+/// (indexed by i).
+///
+/// The variants are as follows.
+///
+/// Function Variant Use i j
+///
+/// mmul4 A = C = 0, Y = M Montgomery 0 0
+/// dmul4 A = 0 Montgomery 0 +
+/// mmla4 C = 0, Y = M Montgomery + 0
+/// dmla4 exactly as shown Montgomery + +
+/// mont4 U = C = 0, V = M Montgomery any 0
+///
+/// mul4zc U = V = A = C = 0 Plain 0 0
+/// mul4 U = V = A = 0 Plain 0 +
+/// mla4zc U = V = C = 0 Plain + 0
+/// mla4 U = V = 0 Plain + +
+///
+/// The `mmul4' and `mmla4' functions are also responsible for calculating
+/// the Montgomery reduction factor Y = (A + U V) M used by the rest of the
+/// inner loop.
///--------------------------------------------------------------------------
/// Macro definitions.
movdqu [rdi], xmm0
ret
-
ENDFUNC
INTFUNC(dmul4)
movdqu [rdi], xmm6
ret
-
ENDFUNC
INTFUNC(dmla4)
movdqu [rdi], xmm6
ret
-
ENDFUNC
INTFUNC(mul4zc)
movdqu [rdi], xmm6
ret
-
ENDFUNC
INTFUNC(mul4)
movdqu [rdi], xmm6
ret
-
ENDFUNC
INTFUNC(mla4zc)
movdqu [rdi], xmm6
ret
-
ENDFUNC
INTFUNC(mla4)
movdqu [rdi], xmm6
ret
-
ENDFUNC
INTFUNC(mmul4)
// On entry, RDI points to the destination buffer; RAX and RBX point
// to the packed operands U and N; and XMM8/XMM9 and XMM10/XMM11 hold
- // the expanded operands V and M. The stack pointer must be 8 modulo 16
- // (as usual for AMD64 ABIs).
+ // the expanded operands V and M. The stack pointer must be 8 modulo
+ // 16 (as usual for AMD64 ABIs).
//
// On exit, we store Y = U V M mod B in XMM10/XMM11, and write the
// low 128 bits of the sum U V + N Y to [RDI], leaving the remaining
mulcore xmm4, 0, xmm8, xmm9, xmm12, xmm13, xmm14, xmm15
propout xmm7, lo, xmm12, xmm13
jmp 5f
-
ENDFUNC
INTFUNC(mmla4)
movdqu xmm4, [rax]
#if ABI_WIN
stalloc 48 + 8 // space for the carries
-# define STKTMP(i) [rsp + i]
+# define STKTMP(i) [SP + i]
#endif
#if ABI_SYSV
-# define STKTMP(i) [rsp + i - 48 - 8] // use red zone
+# define STKTMP(i) [SP + i - 48 - 8] // use red zone
#endif
endprologue
// And, with that, we're done.
movdqu [rdi], xmm6
ret
-
ENDFUNC
///--------------------------------------------------------------------------
FUNC(mpx_umul4_amd64_sse2)
// void mpx_umul4_amd64_sse2(mpw *dv, const mpw *av, const mpw *avl,
- // const mpw *bv, const mpw *bvl);
+ // const mpw *bv, const mpw *bvl);
// Establish the arguments and do initial setup.
//
endprologue
mov DV, rdi
-
#endif
#if ABI_WIN
endprologue
mov rdi, DV
- mov BVL, [rsp + 224]
-
+ mov BVL, [SP + 224]
#endif
// Prepare for the first iteration.
#endif
#if ABI_WIN
-
rstrxmm xmm6, 0
rstrxmm xmm7, 16
rstrxmm xmm8, 32
stfree 160 + 8
popreg rdi
popreg rbx
-
#endif
ret
endprologue
mov DV, rdi
-
#endif
#if ABI_WIN
endprologue
mov rdi, DV
- mov N, [rsp + 224]
- mov MI, [rsp + 232]
-
+ mov N, [SP + 224]
+ mov MI, [SP + 232]
#endif
// Establish the expanded operands.
#endif
#if ABI_WIN
-
rstrxmm xmm6, 0
rstrxmm xmm7, 16
rstrxmm xmm8, 32
popreg r12
popreg rdi
popreg rbx
-
#endif
ret
// outer loop dv r10 rcx
// outer loop dv limit r11 r11
// nv base rdx r8
- // nv limit r9 r12*
+ // nv limit r9 r10*
// n rcx r9
// c rcx r9
#if ABI_SYSV
-
# define DVL rax
# define DVL4 rsi
# define MI r8
endprologue
mov DV, rdi
-
#endif
#if ABI_WIN
-
# define DVL rax
# define DVL4 rdx
# define MI r10
# define DV rcx
# define DVLO r11
# define NV r8
-# define NVL r12
+# define NVL r10
# define N r9
# define C r9d
pushreg rbx
pushreg rdi
- pushreg r12
- stalloc 160
+ stalloc 168
savexmm xmm6, 0
savexmm xmm7, 16
endprologue
mov rdi, DV
- mov MI, [rsp + 224]
-
+ mov MI, [SP + 224]
#endif
// Establish the expanded operands and the blocks-of-4 dv limit.
cmp rdi, DVL4
jb 0b
- // Deal with the tail end.
+ // Deal with the tail end. Note that the actual destination length
+ // won't be an exacty number of blocks of four, so it's safe to just
+ // drop through here.
7: add [rdi], C
- mov C, 0 // preserves flags
+ mov C, 0
add rdi, 4
adc C, 0
cmp rdi, DVL
jb 7b
- // All done for this iteration. Start the next. (This must have at
- // least one follow-on iteration, or we'd not have started this outer
- // loop.)
-8: mov rdi, DV // -> Z = dv[i]
- mov rbx, NV // -> X = nv[0]
- cmp rdi, DVLO // all done yet?
+ // All done for this iteration. Start the next.
+ cmp DV, DVLO // all done yet?
jae 9f
+ mov rdi, DV // -> Z = dv[i]
+ mov rbx, NV // -> X = nv[0]
add DV, 16
call mont4
add rdi, 16
#endif
#if ABI_WIN
-
rstrxmm xmm6, 0
rstrxmm xmm7, 16
rstrxmm xmm8, 32
rstrxmm xmm14, 128
rstrxmm xmm15, 144
- stfree 160
- popreg r12
+ stfree 168
popreg rdi
popreg rbx
-
#endif
ret
# define ARG8 STKARG(4)
# define STKARG_OFFSET 224
#endif
-#define STKARG(i) [rsp + STKARG_OFFSET + 8*(i)]
+#define STKARG(i) [SP + STKARG_OFFSET + 8*(i)]
// sysv win
// dmul smul mmul mont dmul smul mmul mont
testepilogue
ENDFUNC
+FUNC(test_mul4zc)
+ testprologue smul
+ testldcarry
+ testtop nil
+ call mul4zc
+ testtail
+ testcarryout
+ testepilogue
+ENDFUNC
+
FUNC(test_mla4)
testprologue smul
testldcarry
testepilogue
ENDFUNC
+FUNC(test_mla4zc)
+ testprologue smul
+ testldcarry
+ testtop nil
+ call mla4zc
+ testtail
+ testcarryout
+ testepilogue
+ENDFUNC
+
FUNC(test_mmul4)
testprologue mmul
testtop r11
call mmul4
testtail
+ pshufd xmm10, xmm10, SHUF(0, 2, 1, 3)
+ pshufd xmm11, xmm11, SHUF(0, 2, 1, 3)
movdqu [r10 + 0], xmm10
movdqu [r10 + 16], xmm11
testcarryout
testtop r11
call mmla4
testtail
+ pshufd xmm10, xmm10, SHUF(0, 2, 1, 3)
+ pshufd xmm11, xmm11, SHUF(0, 2, 1, 3)
movdqu [r10 + 0], xmm10
movdqu [r10 + 16], xmm11
testcarryout
testtop
call mont4
testtail
+ pshufd xmm10, xmm10, SHUF(0, 2, 1, 3)
+ pshufd xmm11, xmm11, SHUF(0, 2, 1, 3)
movdqu [r10 + 0], xmm10
movdqu [r10 + 16], xmm11
testcarryout