X-Git-Url: https://git.distorted.org.uk/~mdw/catacomb/blobdiff_plain/0aec06589cca20e77119fd4a6c7fd69999c8ce1f..1b07c4f340465551b7d12c3be3bcecd258a3c32d:/base/dispatch.c diff --git a/base/dispatch.c b/base/dispatch.c index b2a032e5..bda7f88e 100644 --- a/base/dispatch.c +++ b/base/dispatch.c @@ -29,6 +29,7 @@ #include "config.h" +#include #include #include #include @@ -43,109 +44,65 @@ #if CPUFAM_X86 || CPUFAM_AMD64 -# define EFLAGS_ID (1u << 21) +enum { + CPUID_1_D, /* eax = 1 => edx&?? */ # define CPUID1D_SSE2 (1u << 26) # define CPUID1D_FXSR (1u << 24) -# define CPUID1C_AESNI (1u << 25) -struct cpuid { unsigned a, b, c, d; }; + CPUID_1_C, /* eax = 1 => ecx&?? */ +# define CPUID1C_PCLMUL (1u << 1) +# define CPUID1C_SSSE3 (1u << 9) +# define CPUID1C_AESNI (1u << 25) +# define CPUID1C_AVX (1u << 28) +# define CPUID1C_RDRAND (1u << 30) -/* --- @cpuid@ --- * - * - * Arguments: @struct cpuid *cc@ = where to write the result - * @unsigned a, c@ = EAX and ECX registers to set - * - * Returns: --- - * - * Use: Minimal C wrapper around the x86 `CPUID' instruction. Checks - * that the instruction is actually available before invoking - * it; fills the output structure with zero if it's not going to - * work. - */ +}; -#ifdef __GNUC__ -# if CPUFAM_X86 -static __inline__ unsigned getflags(void) - { unsigned f; __asm__ ("pushf; popl %0" : "=g" (f)); return (f); } -static __inline__ unsigned setflags(unsigned f) -{ - unsigned ff; - __asm__ ("pushf; pushl %1; popf; pushf; popl %0; popf" - : "=g" (ff) - : "g" (f)); - return (ff); -} -# else -static __inline__ unsigned long getflags(void) - { unsigned long f; __asm__ ("pushf; popq %0" : "=g" (f)); return (f); } -static __inline__ unsigned long long setflags(unsigned long f) -{ - unsigned long ff; - __asm__ ("pushf; pushq %1; popf; pushf; popq %0; popf" - : "=g" (ff) - : "g" (f)); - return (ff); -} -# endif -#endif +struct cpuid { unsigned a, b, c, d; }; +extern int dispatch_x86ish_cpuid(struct cpuid *, unsigned a, unsigned c); +extern int dispatch_x86ish_xmmregisters_p(void); +extern int dispatch_x86ish_rdrand(unsigned op, unsigned *); static void cpuid(struct cpuid *cc, unsigned a, unsigned c) { -#ifdef __GNUC__ - unsigned f; -#endif - - cc->a = cc->b = cc->c = cc->d = 0; - -#ifdef __GNUC__ - /* Stupid dance to detect whether the CPUID instruction is available. */ - f = getflags(); - if (!(setflags(f | EFLAGS_ID) & EFLAGS_ID) || - setflags(f & ~EFLAGS_ID) & EFLAGS_ID) { + int rc = dispatch_x86ish_cpuid(cc, a, c); + if (rc) dispatch_debug("CPUID instruction not available"); - return; - } - setflags(f); - - /* Alas, EBX is magical in PIC code, so abuse ESI instead. This isn't - * pretty, but it works. - */ -# if CPUFAM_X86 - __asm__ ("pushl %%ebx; cpuid; movl %%ebx, %%esi; popl %%ebx" - : "=a" (cc->a), "=S" (cc->b), "=c" (cc->c), "=d" (cc->d) - : "a" (a) , "c" (c)); -# elif CPUFAM_AMD64 - __asm__ ("pushq %%rbx; cpuid; movl %%ebx, %%esi; popq %%rbx" - : "=a" (cc->a), "=S" (cc->b), "=c" (cc->c), "=d" (cc->d) - : "a" (a) , "c" (c)); -# else -# error "I'm confused." -# endif - dispatch_debug("CPUID(%08x, %08x) -> %08x, %08x, %08x, %08x", - a, c, cc->a, cc->b, cc->c, cc->d); -#else - dispatch_debug("GNU inline assembler not available; can't CPUID"); -#endif + else + dispatch_debug("CPUID(%08x, %08x) -> %08x, %08x, %08x, %08x", + a, c, cc->a, cc->b, cc->c, cc->d); } static unsigned cpuid_maxleaf(void) { struct cpuid c; cpuid(&c, 0, 0); return (c.a); } -/* --- @cpuid_features_p@ --- * +/* --- @cpuid_feature_p@ --- * * - * Arguments: @unsigned dbits@ = bits to check in EDX - * @unsigned cbits@ = bits to check in ECX + * Arguments: @unsigned leaf@ = leaf to look up + * @unsigned bits@ = bits to check * - * Returns: Nonzero if all the requested bits are set in the CPUID result - * on leaf 1. + * Returns: Nonzero if all the requested bits are set in the requested + * CPUID result. */ -static int cpuid_features_p(unsigned dbits, unsigned cbits) +static int cpuid_feature_p(unsigned leaf, unsigned bits) { struct cpuid c; - if (cpuid_maxleaf() < 1) return (0); - cpuid(&c, 1, 0); - return ((c.d & dbits) == dbits && (c.c & cbits) == cbits); + unsigned r; + + switch (leaf) { + case CPUID_1_D: + if (cpuid_maxleaf() < 1) return (0); + cpuid(&c, 1, 0); r = c.d; + break; + case CPUID_1_C: + if (cpuid_maxleaf() < 1) return (0); + cpuid(&c, 1, 0); r = c.c; + break; + default: + assert(!"unknown cpuid leaf"); + } + return ((r&bits) == bits); } /* --- @xmm_registers_available_p@ --- * @@ -158,43 +115,52 @@ static int cpuid_features_p(unsigned dbits, unsigned cbits) static int xmm_registers_available_p(void) { -#ifdef __GNUC__ - unsigned f; - /* This hack is by Agner Fog. Use FXSAVE/FXRSTOR to figure out whether the - * XMM registers are actually alive. - */ - if (!cpuid_features_p(CPUID1D_FXSR, 0)) return (0); -# if CPUFAM_X86 - __asm__ ("movl %%esp, %%edx; subl $512, %%esp; andl $~15, %%esp\n" - "fxsave (%%esp)\n" - "movl 160(%%esp), %%eax; xorl $0xaaaa5555, 160(%%esp)\n" - "fxrstor (%%esp); fxsave (%%esp)\n" - "movl 160(%%esp), %%ecx; movl %%eax, 160(%%esp)\n" - "fxrstor (%%esp); movl %%edx, %%esp\n" - "xorl %%ecx, %%eax" - : "=a" (f) - : /* no inputs */ - : "%ecx", "%edx"); -# elif CPUFAM_AMD64 - __asm__ ("movq %%rsp, %%rdx; subq $512, %%rsp; andq $~15, %%rsp\n" - "fxsave (%%rsp)\n" - "movl 160(%%rsp), %%eax; xorl $0xaaaa5555, 160(%%rsp)\n" - "fxrstor (%%rsp); fxsave (%%rsp)\n" - "movl 160(%%rsp), %%ecx; movl %%eax, 160(%%rsp)\n" - "fxrstor (%%rsp); movq %%rdx, %%rsp\n" - "xorl %%ecx, %%eax" - : "=a" (f) - : /* no inputs */ - : "%ecx", "%rdx"); -# else -# error "I'm confused." -# endif + int f = dispatch_x86ish_xmmregisters_p(); + dispatch_debug("XMM registers %savailable", f ? "" : "not "); return (f); -#else - dispatch_debug("GNU inline assembler not available; can't check for XMM"); +} + +/* --- @rdrand_works_p@ --- * + * + * + * Arguments: --- + * + * Returns: Nonzero if the `rdrand' instruction actually works. Assumes + * that it's already been verified to be safe to issue. + */ + +enum { OP_RDRAND, OP_RDSEED }; + +static int rdrand_works_p(unsigned op) +{ + unsigned ref, x, i; + const char *what; + + switch (op) { + case OP_RDRAND: what = "RDRAND"; break; + default: assert(!"unexpected op"); + } + + /* Check that it doesn't always give the same answer. Try four times: this + * will fail with probability %$2^{-128}$% with a truly random generator, + * which seems fair enough. + */ + if (dispatch_x86ish_rdrand(op, &ref)) goto fail; + for (i = 0; i < 4; i++) { + if (dispatch_x86ish_rdrand(op, &x)) goto fail; + if (x != ref) goto not_stuck; + } + dispatch_debug("%s always returns 0x%08x!", what, ref); + return (0); + +not_stuck: + dispatch_debug("%s instruction looks plausible", what); + return (1); + +fail: + dispatch_debug("%s instruction fails too often", what); return (0); -#endif } #endif @@ -204,13 +170,12 @@ static int xmm_registers_available_p(void) /* Try to find the system's definitions for auxiliary vector entries. */ #ifdef HAVE_SYS_AUXV_H # include -#else -# ifdef HAVE_LINUX_AUXVEC_H -# include -# endif -# ifdef HAVE_ASM_HWCAP_H -# include -# endif +#endif +#ifdef HAVE_LINUX_AUXVEC_H +# include +#endif +#ifdef HAVE_ASM_HWCAP_H +# include #endif /* The type of entries in the auxiliary vector. I'm assuming that `unsigned @@ -229,6 +194,16 @@ struct auxentry { unsigned long type; union auxval value; }; # define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap) #endif +#if defined(AT_HWCAP) && CPUFAM_ARM64 +# define WANT_ANY 1 +# define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap) +#endif + +#if defined(AT_HWCAP2) && CPUFAM_ARMEL +# define WANT_ANY 1 +# define WANT_AT_HWCAP2(_) _(AT_HWCAP2, u, hwcap2) +#endif + /* If we couldn't find any interesting entries then we can switch all of this * machinery off. Also do that if we have no means for atomic updates. */ @@ -246,6 +221,9 @@ static unsigned hwcaps = 0; #ifndef WANT_AT_HWCAP # define WANT_AT_HWCAP(_) #endif +#ifndef WANT_AT_HWCAP2 +# define WANT_AT_HWCAP2(_) +#endif /* For each CPU family, define two lists. * @@ -261,12 +239,23 @@ static unsigned hwcaps = 0; */ #if CPUFAM_ARMEL # define WANTAUX(_) \ - WANT_AT_HWCAP(_) + WANT_AT_HWCAP(_) \ + WANT_AT_HWCAP2(_) # define CAPMAP(_) \ _(ARM_VFP, "arm:vfp") \ _(ARM_NEON, "arm:neon") \ _(ARM_V4, "arm:v4") \ - _(ARM_D32, "arm:d32") + _(ARM_D32, "arm:d32") \ + _(ARM_AES, "arm:aes") \ + _(ARM_PMULL, "arm:pmull") +#endif +#if CPUFAM_ARM64 +# define WANTAUX(_) \ + WANT_AT_HWCAP(_) +# define CAPMAP(_) \ + _(ARM_NEON, "arm:neon") \ + _(ARM_AES, "arm:aes") \ + _(ARM_PMULL, "arm:pmull") #endif /* Build the bitmask for `hwcaps' from the `CAPMAP' list. */ @@ -319,8 +308,8 @@ static void probe_hwcaps(void) /* Shiny new libc lets us request individual entry types. This is almost * too easy. */ -# define CAP__GET(type, slot, ubranch) \ - probed.slot.ubranch = (AUXUTYPE_##ubranch)getauxval(type); +# define CAP__GET(type, ubranch, slot) \ + probed.slot = (AUXUTYPE_##ubranch)getauxval(type); WANTAUX(CAP__GET) #else /* Otherwise we're a bit stuck, really. Modern Linux kernels make a copy @@ -377,6 +366,17 @@ static void probe_hwcaps(void) if (probed.hwcap & HWCAP_NEON) hw |= HF_ARM_NEON; if (probed.hwcap & HWCAP_VFPD32) hw |= HF_ARM_D32; if (probed.hwcap & HWCAP_VFPv4) hw |= HF_ARM_V4; +# ifdef HWCAP2_AES + if (probed.hwcap2 & HWCAP2_AES) hw |= HF_ARM_AES; +# endif +# ifdef HWCAP2_PMULL + if (probed.hwcap2 & HWCAP2_PMULL) hw |= HF_ARM_PMULL; +# endif +#endif +#if CPUFAM_ARM64 + if (probed.hwcap & HWCAP_ASIMD) hw |= HF_ARM_NEON; + if (probed.hwcap & HWCAP_AES) hw |= HF_ARM_AES; + if (probed.hwcap & HWCAP_PMULL) hw |= HF_ARM_PMULL; #endif /* Store the bitmask of features we probed for everyone to see. */ @@ -461,14 +461,14 @@ static int IGNORABLE check_env(const char *ftok) if (!p) return (-1); for (;;) { - while (isspace((unsigned char)*p)) p++; + while (ISSPACE(*p)) p++; if (!*p) return (-1); switch (*p) { case '+': d = +1; p++; break; case '-': d = 0; p++; break; default: d = -1; break; } - for (q = p; *q && !isspace((unsigned char)*q); q++); + for (q = p; *q && !ISSPACE(*q); q++); if (d >= 0) { for (pp = ftok; p < q && *pp && *p == *pp; p++, pp++); if ((p == q && !*pp) || (*p == '*' && p + 1 == q)) return (d); @@ -502,8 +502,7 @@ int cpu_feature_p(int feat) int IGNORABLE f; IGNORE(f); #define CASE_CPUFEAT(feat, ftok, cond) case CPUFEAT_##feat: \ - if ((f = feat_debug(ftok, "environment override", \ - check_env(ftok))) >= 0) \ + if ((f = feat_debug(ftok, "environment override", check_env(ftok))) >= 0) \ return (f); \ else \ return (feat_debug(ftok, "runtime probe", cond)); @@ -511,11 +510,23 @@ int cpu_feature_p(int feat) switch (feat) { #if CPUFAM_X86 || CPUFAM_AMD64 CASE_CPUFEAT(X86_SSE2, "x86:sse2", - xmm_registers_available_p() && - cpuid_features_p(CPUID1D_SSE2, 0)); + cpuid_feature_p(CPUID_1_D, CPUID1D_SSE2) && + xmm_registers_available_p()); CASE_CPUFEAT(X86_AESNI, "x86:aesni", - xmm_registers_available_p() && - cpuid_features_p(CPUID1D_SSE2, CPUID1C_AESNI)); + cpuid_feature_p(CPUID_1_D, CPUID1C_AESNI) && + xmm_registers_available_p()); + CASE_CPUFEAT(X86_RDRAND, "x86:rdrand", + cpuid_feature_p(CPUID_1_C, CPUID1C_RDRAND) && + rdrand_works_p(OP_RDRAND)); + CASE_CPUFEAT(X86_AVX, "x86:avx", + cpuid_feature_p(CPUID_1_C, CPUID1C_AVX) && + xmm_registers_available_p()); + CASE_CPUFEAT(X86_SSSE3, "x86:ssse3", + cpuid_feature_p(CPUID_1_C, CPUID1C_SSSE3) && + xmm_registers_available_p()); + CASE_CPUFEAT(X86_PCLMUL, "x86:pclmul", + cpuid_feature_p(CPUID_1_C, CPUID1C_PCLMUL) && + xmm_registers_available_p()); #endif #ifdef CAPMAP # define FEATP__CASE(feat, tok) \