X-Git-Url: https://git.distorted.org.uk/~mdw/catacomb/blobdiff_plain/119bdfc4c80e313417047de10700e6b8c3955d0d..00b52725bf15e844360d07248de1d8cbbc1e165e:/base/dispatch.c diff --git a/base/dispatch.c b/base/dispatch.c index 459302f0..f1859c2b 100644 --- a/base/dispatch.c +++ b/base/dispatch.c @@ -29,6 +29,7 @@ #include "config.h" +#include #include #include #include @@ -43,17 +44,30 @@ #if CPUFAM_X86 || CPUFAM_AMD64 +enum { + CPUID_1_D, /* eax = 1 => edx&?? */ # define CPUID1D_SSE2 (1u << 26) # define CPUID1D_FXSR (1u << 24) + + CPUID_1_C, /* eax = 1 => ecx&?? */ # define CPUID1C_PCLMUL (1u << 1) # define CPUID1C_SSSE3 (1u << 9) # define CPUID1C_AESNI (1u << 25) +# define CPUID1C_OSXSAVE (1u << 27) # define CPUID1C_AVX (1u << 28) # define CPUID1C_RDRAND (1u << 30) + CPUID_7_0_B, /* eax = 7, ecx = 0 => ebx&?? */ +# define CPUID70B_AVX2 (1u << 5) +# define CPUID70B_RDSEED (1u << 18) +}; + struct cpuid { unsigned a, b, c, d; }; +struct xcr { unsigned lo, hi; }; extern int dispatch_x86ish_cpuid(struct cpuid *, unsigned a, unsigned c); extern int dispatch_x86ish_xmmregisters_p(void); +extern int dispatch_x86ish_xgetbv(struct xcr *z_out, unsigned c); +extern int dispatch_x86ish_rdrand(unsigned op, unsigned *); static void cpuid(struct cpuid *cc, unsigned a, unsigned c) { @@ -68,29 +82,45 @@ static void cpuid(struct cpuid *cc, unsigned a, unsigned c) static unsigned cpuid_maxleaf(void) { struct cpuid c; cpuid(&c, 0, 0); return (c.a); } -/* --- @cpuid_features_p@ --- * +/* --- @cpuid_feature_p@ --- * * - * Arguments: @unsigned dbits@ = bits to check in EDX - * @unsigned cbits@ = bits to check in ECX + * Arguments: @unsigned leaf@ = leaf to look up + * @unsigned bits@ = bits to check * - * Returns: Nonzero if all the requested bits are set in the CPUID result - * on leaf 1. + * Returns: Nonzero if all the requested bits are set in the requested + * CPUID result. */ -static int cpuid_features_p(unsigned dbits, unsigned cbits) +static int cpuid_feature_p(unsigned leaf, unsigned bits) { struct cpuid c; - if (cpuid_maxleaf() < 1) return (0); - cpuid(&c, 1, 0); - return ((c.d & dbits) == dbits && (c.c & cbits) == cbits); + unsigned r; + + switch (leaf) { + case CPUID_1_D: + if (cpuid_maxleaf() < 1) return (0); + cpuid(&c, 1, 0); r = c.d; + break; + case CPUID_1_C: + if (cpuid_maxleaf() < 1) return (0); + cpuid(&c, 1, 0); r = c.c; + break; + case CPUID_7_0_B: + if (cpuid_maxleaf() < 7) return (0); + cpuid(&c, 7, 0); r = c.b; + break; + default: + assert(!"unknown cpuid leaf"); + } + return ((r&bits) == bits); } -/* --- @xmm_registers_available_p@ --- * +/* --- @{x,y}mm_registers_available_p@ --- * * * Arguments: --- * - * Returns: Nonzero if the operating system has made the XMM registers - * available for use. + * Returns: Nonzero if the operating system has made the XMM or YMM + * registers available for use. */ static int xmm_registers_available_p(void) @@ -101,6 +131,22 @@ static int xmm_registers_available_p(void) return (f); } +static int ymm_registers_available_p(void) +{ + struct xcr xcr0; + int f; + + f = cpuid_feature_p(CPUID_1_C, CPUID1C_OSXSAVE); + dispatch_debug("XGETBV %savailable", f ? "" : "not "); + if (!f) return (0); + + dispatch_x86ish_xgetbv(&xcr0, 0); f = (xcr0.lo&0x06) == 0x06; + dispatch_debug("YMM state %senabled", f ? "" : "not "); + if (!f) return (0); + + return (1); +} + /* --- @rdrand_works_p@ --- * * * @@ -110,47 +156,37 @@ static int xmm_registers_available_p(void) * that it's already been verified to be safe to issue. */ -#ifdef __GNUC__ -static int rdrand(unsigned *x) -{ - int i, rc; - unsigned _t; - - i = 16; - __asm__ ("" : "=g" (_t)); - __asm__ ("0: rdrand %2; jc 1f; decl %1; jnz 0b\n" - "mov $-1, %0; jmp 9f\n" - "1: movl %2, (%3); xorl %0, %0\n" - "9:" - : "=r" (rc), "+r" (i), "+r" (_t) - : "r" (x) - : "cc"); - return (rc); -} -#endif +enum { OP_RDRAND, OP_RDSEED }; -static int rdrand_works_p(void) +static int rdrand_works_p(unsigned op) { unsigned ref, x, i; + const char *what; + + switch (op) { + case OP_RDRAND: what = "RDRAND"; break; + case OP_RDSEED: what = "RDSEED"; break; + default: assert(!"unexpected op"); + } /* Check that it doesn't always give the same answer. Try four times: this * will fail with probability %$2^{-128}$% with a truly random generator, * which seems fair enough. */ - if (rdrand(&ref)) goto fail; + if (dispatch_x86ish_rdrand(op, &ref)) goto fail; for (i = 0; i < 4; i++) { - if (rdrand(&x)) goto fail; + if (dispatch_x86ish_rdrand(op, &x)) goto fail; if (x != ref) goto not_stuck; } - dispatch_debug("RDRAND always returns 0x%08x!", ref); + dispatch_debug("%s always returns 0x%08x!", what, ref); return (0); not_stuck: - dispatch_debug("RDRAND instruction looks plausible"); + dispatch_debug("%s instruction looks plausible", what); return (1); fail: - dispatch_debug("RDRAND instruction fails too often"); + dispatch_debug("%s instruction fails too often", what); return (0); } @@ -244,6 +280,7 @@ static unsigned hwcaps = 0; # define WANTAUX(_) \ WANT_AT_HWCAP(_) # define CAPMAP(_) \ + _(ARM_NEON, "arm:neon") \ _(ARM_AES, "arm:aes") \ _(ARM_PMULL, "arm:pmull") #endif @@ -364,6 +401,7 @@ static void probe_hwcaps(void) # endif #endif #if CPUFAM_ARM64 + if (probed.hwcap & HWCAP_ASIMD) hw |= HF_ARM_NEON; if (probed.hwcap & HWCAP_AES) hw |= HF_ARM_AES; if (probed.hwcap & HWCAP_PMULL) hw |= HF_ARM_PMULL; #endif @@ -395,7 +433,7 @@ static unsigned get_hwcaps(void) unsigned hw; DISPATCH_LOAD(hwcaps, hw); - if (!(hwcaps & HF_PROBED)) { probe_hwcaps(); DISPATCH_LOAD(hwcaps, hw); } + if (!(hw & HF_PROBED)) { probe_hwcaps(); DISPATCH_LOAD(hwcaps, hw); } return (hw); } @@ -450,14 +488,14 @@ static int IGNORABLE check_env(const char *ftok) if (!p) return (-1); for (;;) { - while (isspace((unsigned char)*p)) p++; + while (ISSPACE(*p)) p++; if (!*p) return (-1); switch (*p) { case '+': d = +1; p++; break; case '-': d = 0; p++; break; default: d = -1; break; } - for (q = p; *q && !isspace((unsigned char)*q); q++); + for (q = p; *q && !ISSPACE(*q); q++); if (d >= 0) { for (pp = ftok; p < q && *pp && *p == *pp; p++, pp++); if ((p == q && !*pp) || (*p == '*' && p + 1 == q)) return (d); @@ -491,8 +529,7 @@ int cpu_feature_p(int feat) int IGNORABLE f; IGNORE(f); #define CASE_CPUFEAT(feat, ftok, cond) case CPUFEAT_##feat: \ - if ((f = feat_debug(ftok, "environment override", \ - check_env(ftok))) >= 0) \ + if ((f = feat_debug(ftok, "environment override", check_env(ftok))) >= 0) \ return (f); \ else \ return (feat_debug(ftok, "runtime probe", cond)); @@ -500,22 +537,30 @@ int cpu_feature_p(int feat) switch (feat) { #if CPUFAM_X86 || CPUFAM_AMD64 CASE_CPUFEAT(X86_SSE2, "x86:sse2", - cpuid_features_p(CPUID1D_SSE2, 0) && + cpuid_feature_p(CPUID_1_D, CPUID1D_SSE2) && xmm_registers_available_p()); CASE_CPUFEAT(X86_AESNI, "x86:aesni", - cpuid_features_p(CPUID1D_SSE2, CPUID1C_AESNI) && + cpuid_feature_p(CPUID_1_C, CPUID1C_AESNI) && xmm_registers_available_p()); CASE_CPUFEAT(X86_RDRAND, "x86:rdrand", - cpuid_features_p(0, CPUID1C_RDRAND) && rdrand_works_p()); + cpuid_feature_p(CPUID_1_C, CPUID1C_RDRAND) && + rdrand_works_p(OP_RDRAND)); CASE_CPUFEAT(X86_AVX, "x86:avx", - cpuid_features_p(0, CPUID1C_AVX) && - xmm_registers_available_p()); + cpuid_feature_p(CPUID_1_C, CPUID1C_AVX) && + ymm_registers_available_p()); + CASE_CPUFEAT(X86_AVX2, "x86:avx2", + cpuid_feature_p(CPUID_1_C, CPUID1C_AVX) && + cpuid_feature_p(CPUID_7_0_B, CPUID70B_AVX2) && + ymm_registers_available_p()); CASE_CPUFEAT(X86_SSSE3, "x86:ssse3", - cpuid_features_p(0, CPUID1C_SSSE3) && + cpuid_feature_p(CPUID_1_C, CPUID1C_SSSE3) && xmm_registers_available_p()); CASE_CPUFEAT(X86_PCLMUL, "x86:pclmul", - cpuid_features_p(0, CPUID1C_PCLMUL) && + cpuid_feature_p(CPUID_1_C, CPUID1C_PCLMUL) && xmm_registers_available_p()); + CASE_CPUFEAT(X86_RDSEED, "x86:rdseed", + cpuid_feature_p(CPUID_7_0_B, CPUID70B_RDSEED) && + rdrand_works_p(OP_RDSEED)); #endif #ifdef CAPMAP # define FEATP__CASE(feat, tok) \