+#define MAYBE_UMUL4(impl) \
+ extern void mpx_umul4_##impl(mpw */*dv*/, \
+ const mpw */*av*/, const mpw */*avl*/, \
+ const mpw */*bv*/, const mpw */*bvl*/); \
+ static void maybe_umul4_##impl(mpw *dv, mpw *dvl, \
+ const mpw *av, const mpw *avl, \
+ const mpw *bv, const mpw *bvl) \
+ { \
+ size_t an = avl - av, bn = bvl - bv, dn = dvl - dv; \
+ if (!an || an%4 != 0 || !bn || bn%4 != 0 || dn < an + bn) \
+ simple_umul(dv, dvl, av, avl, bv, bvl); \
+ else { \
+ mpx_umul4_##impl(dv, av, avl, bv, bvl); \
+ MPX_ZERO(dv + an + bn, dvl); \
+ } \
+ }
+
+#if CPUFAM_X86
+ MAYBE_UMUL4(x86_sse2)
+ MAYBE_UMUL4(x86_avx)
+#endif
+
+#if CPUFAM_AMD64
+ MAYBE_UMUL4(amd64_sse2)
+ MAYBE_UMUL4(amd64_avx)
+#endif
+
+#if CPUFAM_ARMEL
+ MAYBE_UMUL4(arm_neon)
+#endif
+
+#if CPUFAM_ARM64
+ MAYBE_UMUL4(arm64_simd)
+#endif
+
+static mpx_umul__functype *pick_umul(void)
+{
+#if CPUFAM_X86
+ DISPATCH_PICK_COND(mpx_umul, maybe_umul4_x86_avx,
+ cpu_feature_p(CPUFEAT_X86_AVX));
+ DISPATCH_PICK_COND(mpx_umul, maybe_umul4_x86_sse2,
+ cpu_feature_p(CPUFEAT_X86_SSE2));
+#endif
+#if CPUFAM_AMD64
+ DISPATCH_PICK_COND(mpx_umul, maybe_umul4_amd64_avx,
+ cpu_feature_p(CPUFEAT_X86_AVX));
+ DISPATCH_PICK_COND(mpx_umul, maybe_umul4_amd64_sse2,
+ cpu_feature_p(CPUFEAT_X86_SSE2));
+#endif
+#if CPUFAM_ARMEL
+ DISPATCH_PICK_COND(mpx_umul, maybe_umul4_arm_neon,
+ cpu_feature_p(CPUFEAT_ARM_NEON));
+#endif
+#if CPUFAM_ARM64
+ DISPATCH_PICK_COND(mpx_umul, maybe_umul4_arm64_simd,
+ cpu_feature_p(CPUFEAT_ARM_NEON));
+#endif
+ DISPATCH_PICK_FALLBACK(mpx_umul, simple_umul);
+}
+