X-Git-Url: https://git.distorted.org.uk/~mdw/catacomb/blobdiff_plain/226639f3441312d535bccf45b1c0d15d0ae156a1..89717a56084f7cac56330c8527fbaff99b15709b:/base/dispatch.c diff --git a/base/dispatch.c b/base/dispatch.c index fe3dcfb8..abd019f6 100644 --- a/base/dispatch.c +++ b/base/dispatch.c @@ -30,6 +30,8 @@ #include "config.h" #include +#include +#include #include #include @@ -37,14 +39,18 @@ #include "dispatch.h" -/*----- Main code ---------------------------------------------------------*/ +/*----- Intel x86/AMD64 feature probing -----------------------------------*/ -#ifdef CPUFAM_X86 +#if CPUFAM_X86 || CPUFAM_AMD64 -#define EFLAGS_ID (1u << 21) -#define CPUID1D_SSE2 (1u << 26) -#define CPUID1D_FXSR (1u << 24) -#define CPUID1C_AESNI (1u << 25) +# define EFLAGS_ID (1u << 21) +# define CPUID1D_SSE2 (1u << 26) +# define CPUID1D_FXSR (1u << 24) +# define CPUID1C_PCLMUL (1u << 1) +# define CPUID1C_SSSE3 (1u << 9) +# define CPUID1C_AESNI (1u << 25) +# define CPUID1C_AVX (1u << 28) +# define CPUID1C_RDRAND (1u << 30) struct cpuid { unsigned a, b, c, d; }; @@ -62,16 +68,29 @@ struct cpuid { unsigned a, b, c, d; }; */ #ifdef __GNUC__ +# if CPUFAM_X86 static __inline__ unsigned getflags(void) { unsigned f; __asm__ ("pushf; popl %0" : "=g" (f)); return (f); } static __inline__ unsigned setflags(unsigned f) { unsigned ff; __asm__ ("pushf; pushl %1; popf; pushf; popl %0; popf" - : "=g" (ff) - : "g" (f)); + : "=r" (ff) + : "r" (f)); + return (ff); +} +# else +static __inline__ unsigned long getflags(void) + { unsigned long f; __asm__ ("pushf; popq %0" : "=g" (f)); return (f); } +static __inline__ unsigned long long setflags(unsigned long f) +{ + unsigned long ff; + __asm__ ("pushf; pushq %1; popf; pushf; popq %0; popf" + : "=r" (ff) + : "r" (f)); return (ff); } +# endif #endif static void cpuid(struct cpuid *cc, unsigned a, unsigned c) @@ -85,22 +104,46 @@ static void cpuid(struct cpuid *cc, unsigned a, unsigned c) #ifdef __GNUC__ /* Stupid dance to detect whether the CPUID instruction is available. */ f = getflags(); - if (!(setflags(f | EFLAGS_ID) & EFLAGS_ID)) return; - if ( setflags(f & ~EFLAGS_ID) & EFLAGS_ID ) return; + if (!(setflags(f | EFLAGS_ID) & EFLAGS_ID) || + setflags(f & ~EFLAGS_ID) & EFLAGS_ID) { + dispatch_debug("CPUID instruction not available"); + return; + } setflags(f); /* Alas, EBX is magical in PIC code, so abuse ESI instead. This isn't * pretty, but it works. */ +# if CPUFAM_X86 __asm__ ("pushl %%ebx; cpuid; movl %%ebx, %%esi; popl %%ebx" : "=a" (cc->a), "=S" (cc->b), "=c" (cc->c), "=d" (cc->d) : "a" (a) , "c" (c)); +# elif CPUFAM_AMD64 + __asm__ ("pushq %%rbx; cpuid; movl %%ebx, %%esi; popq %%rbx" + : "=a" (cc->a), "=S" (cc->b), "=c" (cc->c), "=d" (cc->d) + : "a" (a) , "c" (c)); +# else +# error "I'm confused." +# endif + dispatch_debug("CPUID(%08x, %08x) -> %08x, %08x, %08x, %08x", + a, c, cc->a, cc->b, cc->c, cc->d); +#else + dispatch_debug("GNU inline assembler not available; can't CPUID"); #endif } static unsigned cpuid_maxleaf(void) { struct cpuid c; cpuid(&c, 0, 0); return (c.a); } +/* --- @cpuid_features_p@ --- * + * + * Arguments: @unsigned dbits@ = bits to check in EDX + * @unsigned cbits@ = bits to check in ECX + * + * Returns: Nonzero if all the requested bits are set in the CPUID result + * on leaf 1. + */ + static int cpuid_features_p(unsigned dbits, unsigned cbits) { struct cpuid c; @@ -109,6 +152,14 @@ static int cpuid_features_p(unsigned dbits, unsigned cbits) return ((c.d & dbits) == dbits && (c.c & cbits) == cbits); } +/* --- @xmm_registers_available_p@ --- * + * + * Arguments: --- + * + * Returns: Nonzero if the operating system has made the XMM registers + * available for use. + */ + static int xmm_registers_available_p(void) { #ifdef __GNUC__ @@ -117,6 +168,7 @@ static int xmm_registers_available_p(void) * XMM registers are actually alive. */ if (!cpuid_features_p(CPUID1D_FXSR, 0)) return (0); +# if CPUFAM_X86 __asm__ ("movl %%esp, %%edx; subl $512, %%esp; andl $~15, %%esp\n" "fxsave (%%esp)\n" "movl 160(%%esp), %%eax; xorl $0xaaaa5555, 160(%%esp)\n" @@ -127,14 +179,355 @@ static int xmm_registers_available_p(void) : "=a" (f) : /* no inputs */ : "%ecx", "%edx"); +# elif CPUFAM_AMD64 + __asm__ ("movq %%rsp, %%rdx; subq $512, %%rsp; andq $~15, %%rsp\n" + "fxsave (%%rsp)\n" + "movl 160(%%rsp), %%eax; xorl $0xaaaa5555, 160(%%rsp)\n" + "fxrstor (%%rsp); fxsave (%%rsp)\n" + "movl 160(%%rsp), %%ecx; movl %%eax, 160(%%rsp)\n" + "fxrstor (%%rsp); movq %%rdx, %%rsp\n" + "xorl %%ecx, %%eax" + : "=a" (f) + : /* no inputs */ + : "%ecx", "%rdx"); +# else +# error "I'm confused." +# endif + dispatch_debug("XMM registers %savailable", f ? "" : "not "); return (f); #else + dispatch_debug("GNU inline assembler not available; can't check for XMM"); + return (0); +#endif +} + +/* --- @rdrand_works_p@ --- * + * + * + * Arguments: --- + * + * Returns: Nonzero if the `rdrand' instruction actually works. Assumes + * that it's already been verified to be safe to issue. + */ + +#ifdef __GNUC__ +static int rdrand(unsigned *x) +{ + int i, rc; + unsigned _t; + + i = 16; + __asm__ ("" : "=g" (_t)); + __asm__ ("0: rdrand %2; jc 1f; decl %1; jnz 0b\n" + "mov $-1, %0; jmp 9f\n" + "1: movl %2, (%3); xorl %0, %0\n" + "9:" + : "=r" (rc), "+r" (i), "+r" (_t) + : "r" (x) + : "cc"); + return (rc); +} +#endif + +static int rdrand_works_p(void) +{ + unsigned ref, x, i; + + /* Check that it doesn't always give the same answer. Try four times: this + * will fail with probability %$2^{-128}$% with a truly random generator, + * which seems fair enough. + */ + if (rdrand(&ref)) goto fail; + for (i = 0; i < 4; i++) { + if (rdrand(&x)) goto fail; + if (x != ref) goto not_stuck; + } + dispatch_debug("RDRAND always returns 0x%08x!", ref); + return (0); + +not_stuck: + dispatch_debug("RDRAND instruction looks plausible"); + return (1); + +fail: + dispatch_debug("RDRAND instruction fails too often"); return (0); +} + +#endif + +/*----- General feature probing using auxiliary vectors -------------------*/ + +/* Try to find the system's definitions for auxiliary vector entries. */ +#ifdef HAVE_SYS_AUXV_H +# include +#endif +#ifdef HAVE_LINUX_AUXVEC_H +# include +#endif +#ifdef HAVE_ASM_HWCAP_H +# include +#endif + +/* The type of entries in the auxiliary vector. I'm assuming that `unsigned + * long' matches each platform's word length; if this is false then we'll + * need some host-specific tweaking here. + */ +union auxval { long i; unsigned long u; const void *p; }; +struct auxentry { unsigned long type; union auxval value; }; + +/* Register each CPU family's interest in the auxiliary vector. Make sure + * that the necessary entry types are defined. This is primarily ordered by + * entry type to minimize duplication. + */ +#if defined(AT_HWCAP) && CPUFAM_ARMEL +# define WANT_ANY 1 +# define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap) +#endif + +#if defined(AT_HWCAP) && CPUFAM_ARM64 +# define WANT_ANY 1 +# define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap) +#endif + +#if defined(AT_HWCAP2) && CPUFAM_ARMEL +# define WANT_ANY 1 +# define WANT_AT_HWCAP2(_) _(AT_HWCAP2, u, hwcap2) +#endif + +/* If we couldn't find any interesting entries then we can switch all of this + * machinery off. Also do that if we have no means for atomic updates. + */ +#if WANT_ANY && CPU_DISPATCH_P + +/* The main output of this section is a bitmask of detected features. The + * least significant bit will be set if we've tried to probe. Always access + * this using `DISPATCH_LOAD' and `DISPATCH_STORE'. + */ +static unsigned hwcaps = 0; + +/* For each potentially interesting type which turned out not to exist or be + * wanted, define a dummy macro for the sake of the next step. + */ +#ifndef WANT_AT_HWCAP +# define WANT_AT_HWCAP(_) #endif +#ifndef WANT_AT_HWCAP2 +# define WANT_AT_HWCAP2(_) +#endif + +/* For each CPU family, define two lists. + * + * * `WANTAUX' is a list of the `WANT_AT_MUMBLE' macros which the CPU + * family tried to register interest in above. Each entry contains the + * interesting auxiliary vector entry type, the name of the union branch + * for its value, and the name of the slot in `struct auxprobe' in which + * to store the value. + * + * * `CAPMAP' is a list describing the output features which the CPU family + * intends to satisfy from the auxiliary vector. Each entry contains a + * feature name suffix, and the token name (for `check_env'). + */ +#if CPUFAM_ARMEL +# define WANTAUX(_) \ + WANT_AT_HWCAP(_) \ + WANT_AT_HWCAP2(_) +# define CAPMAP(_) \ + _(ARM_VFP, "arm:vfp") \ + _(ARM_NEON, "arm:neon") \ + _(ARM_V4, "arm:v4") \ + _(ARM_D32, "arm:d32") \ + _(ARM_AES, "arm:aes") \ + _(ARM_PMULL, "arm:pmull") +#endif +#if CPUFAM_ARM64 +# define WANTAUX(_) \ + WANT_AT_HWCAP(_) +# define CAPMAP(_) \ + _(ARM_AES, "arm:aes") \ + _(ARM_PMULL, "arm:pmull") +#endif + +/* Build the bitmask for `hwcaps' from the `CAPMAP' list. */ +enum { + HFI_PROBED = 0, +#define HFI__ENUM(feat, tok) HFI_##feat, + CAPMAP(HFI__ENUM) +#undef HFI__ENUM + HFI__END +}; +enum { + HF_PROBED = 1, +#define HF__FLAG(feat, tok) HF_##feat = 1 << HFI_##feat, + CAPMAP(HF__FLAG) +#undef HF__FLAG + HF__END +}; + +/* Build a structure in which we can capture the interesting data from the + * auxiliary vector. + */ +#define AUXUTYPE_i long +#define AUXUTYPE_u unsigned long +#define AUXUTYPE_p const void * +struct auxprobe { +#define AUXPROBE__SLOT(type, ubranch, slot) AUXUTYPE_##ubranch slot; + WANTAUX(AUXPROBE__SLOT) +#undef AUXPROBE_SLOT +}; + +/* --- @probe_hwcaps@ --- * + * + * Arguments: --- + * + * Returns: --- + * + * Use: Attempt to find the auxiliary vector (which is well hidden) + * and discover interesting features from it. + */ + +static void probe_hwcaps(void) +{ + unsigned hw = HF_PROBED; + struct auxprobe probed = { 0 }; + + /* Populate `probed' with the information we manage to retrieve from the + * auxiliary vector. Slots we couldn't find are left zero-valued. + */ +#if defined(HAVE_GETAUXVAL) + /* Shiny new libc lets us request individual entry types. This is almost + * too easy. + */ +# define CAP__GET(type, ubranch, slot) \ + probed.slot = (AUXUTYPE_##ubranch)getauxval(type); + WANTAUX(CAP__GET) +#else + /* Otherwise we're a bit stuck, really. Modern Linux kernels make a copy + * of the vector available in `/procc' so we could try that. + * + * The usual place is stuck on the end of the environment vector, but that + * may well have moved, and we have no way of telling whether it has or + * whether there was ever an auxiliary vector there at all; so don't do + * that. + */ + { + FILE *fp = 0; + unsigned char *p = 0, *q = 0; + const struct auxentry *a; + size_t sz, off, n; + + /* Open the file and read it into a memory chunk. */ + if ((fp = fopen("/proc/self/auxv", "rb")) == 0) goto clean; + sz = 4096; off = 0; + if ((p = malloc(sz)) == 0) goto clean; + for (;;) { + n = fread(p + off, 1, sz - off, fp); + off += n; + if (off < sz) break; + sz *= 2; if ((q = realloc(p, sz)) == 0) break; + p = q; + } + + /* Work through the vector (or as much of it as we found) and extract the + * types we're interested in. + */ + for (a = (const struct auxentry *)p, + n = sz/sizeof(struct auxentry); + n--; a++) { + switch (a->type) { +#define CAP__SWITCH(type, ubranch, slot) \ + case type: probed.slot = a->value.ubranch; break; + WANTAUX(CAP__SWITCH) + case AT_NULL: goto clean; + } + } + + clean: + if (p) free(p); + if (fp) fclose(fp); + } +#endif + + /* Each CPU family now has to pick through what was found and stashed in + * `probed', and set the appropriate flag bits in `hw'. + */ +#if CPUFAM_ARMEL + if (probed.hwcap & HWCAP_VFPv3) hw |= HF_ARM_VFP; + if (probed.hwcap & HWCAP_NEON) hw |= HF_ARM_NEON; + if (probed.hwcap & HWCAP_VFPD32) hw |= HF_ARM_D32; + if (probed.hwcap & HWCAP_VFPv4) hw |= HF_ARM_V4; +# ifdef HWCAP2_AES + if (probed.hwcap2 & HWCAP2_AES) hw |= HF_ARM_AES; +# endif +# ifdef HWCAP2_PMULL + if (probed.hwcap2 & HWCAP2_PMULL) hw |= HF_ARM_PMULL; +# endif +#endif +#if CPUFAM_ARM64 + if (probed.hwcap & HWCAP_AES) hw |= HF_ARM_AES; + if (probed.hwcap & HWCAP_PMULL) hw |= HF_ARM_PMULL; +#endif + + /* Store the bitmask of features we probed for everyone to see. */ + DISPATCH_STORE(hwcaps, hw); + + /* Finally, make a report about the things we found. (Doing this earlier + * will pointlessly widen the window in which multiple threads will do the + * above auxiliary-vector probing.) + */ +#define CAP__DEBUG(feat, tok) \ + dispatch_debug("check auxv for feature `%s': %s", tok, \ + hw & HF_##feat ? "available" : "absent"); + CAPMAP(CAP__DEBUG) +#undef CAP__DEBUG +} + +/* --- @get_hwcaps@ --- * + * + * Arguments: --- + * + * Returns: A mask of hardware capabilities and other features, as probed + * from the auxiliary vector. + */ + +static unsigned get_hwcaps(void) +{ + unsigned hw; + + DISPATCH_LOAD(hwcaps, hw); + if (!(hwcaps & HF_PROBED)) { probe_hwcaps(); DISPATCH_LOAD(hwcaps, hw); } + return (hw); } #endif +/*----- External interface ------------------------------------------------*/ + +/* --- @dispatch_debug@ --- * + * + * Arguments: @const char *fmt@ = a format string + * @...@ = additional arguments + * + * Returns: --- + * + * Use: Writes a formatted message to standard output if dispatch + * debugging is enabled. + */ + +void dispatch_debug(const char *fmt, ...) +{ + va_list ap; + const char *e = getenv("CATACOMB_CPUDISPATCH_DEBUG"); + + if (e && *e != 'n' && *e != '0') { + va_start(ap, fmt); + fputs("Catacomb CPUDISPATCH: ", stderr); + vfprintf(stderr, fmt, ap); + fputc('\n', stderr); + va_end(ap); + } +} + /* --- @check_env@ --- * * * Arguments: @const char *ftok@ = feature token @@ -183,30 +576,58 @@ static int IGNORABLE check_env(const char *ftok) #include +static int IGNORABLE + feat_debug(const char *ftok, const char *check, int verdict) +{ + if (verdict >= 0) { + dispatch_debug("feature `%s': %s -> %s", ftok, check, + verdict ? "available" : "absent"); + } + return (verdict); +} + int cpu_feature_p(int feat) { int IGNORABLE f; IGNORE(f); -#define CHECK_ENV(ftok) \ - do { if ((f = check_env(ftok)) >= 0) return (f); } while (0) +#define CASE_CPUFEAT(feat, ftok, cond) case CPUFEAT_##feat: \ + if ((f = feat_debug(ftok, "environment override", \ + check_env(ftok))) >= 0) \ + return (f); \ + else \ + return (feat_debug(ftok, "runtime probe", cond)); switch (feat) { -#ifdef CPUFAM_X86 - case CPUFEAT_X86_SSE2: { - CHECK_ENV("x86:sse2"); - return (xmm_registers_available_p() && - cpuid_features_p(CPUID1D_SSE2, 0)); - } - case CPUFEAT_X86_AESNI: { - check_env("x86:aesni"); - return (xmm_registers_available_p() && - cpuid_features_p(CPUID1D_SSE2, CPUID1C_AESNI)); - } +#if CPUFAM_X86 || CPUFAM_AMD64 + CASE_CPUFEAT(X86_SSE2, "x86:sse2", + cpuid_features_p(CPUID1D_SSE2, 0) && + xmm_registers_available_p()); + CASE_CPUFEAT(X86_AESNI, "x86:aesni", + cpuid_features_p(CPUID1D_SSE2, CPUID1C_AESNI) && + xmm_registers_available_p()); + CASE_CPUFEAT(X86_RDRAND, "x86:rdrand", + cpuid_features_p(0, CPUID1C_RDRAND) && rdrand_works_p()); + CASE_CPUFEAT(X86_AVX, "x86:avx", + cpuid_features_p(0, CPUID1C_AVX) && + xmm_registers_available_p()); + CASE_CPUFEAT(X86_SSSE3, "x86:ssse3", + cpuid_features_p(0, CPUID1C_SSSE3) && + xmm_registers_available_p()); + CASE_CPUFEAT(X86_PCLMUL, "x86:pclmul", + cpuid_features_p(0, CPUID1C_PCLMUL) && + xmm_registers_available_p()); +#endif +#ifdef CAPMAP +# define FEATP__CASE(feat, tok) \ + CASE_CPUFEAT(feat, tok, get_hwcaps() & HF_##feat) + CAPMAP(FEATP__CASE) +#undef FEATP__CASE #endif default: + dispatch_debug("denying unknown feature %d", feat); return (0); } -#undef CHECK_ENV +#undef CASE_CPUFEAT } /*----- That's all, folks -------------------------------------------------*/