mdw@git.distorted.org.uk Git - catacomb/blob - base/dispatch.c

   1 /* -*-c-*-
   2  *
   3  * CPU-specific dispatch
   4  *
   5  * (c) 2015 Straylight/Edgeware
   6  */
   7
   8 /*----- Licensing notice --------------------------------------------------*
   9  *
  10  * This file is part of Catacomb.
  11  *
  12  * Catacomb is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU Library General Public License as
  14  * published by the Free Software Foundation; either version 2 of the
  15  * License, or (at your option) any later version.
  16  *
  17  * Catacomb is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20  * GNU Library General Public License for more details.
  21  *
  22  * You should have received a copy of the GNU Library General Public
  23  * License along with Catacomb; if not, write to the Free
  24  * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
  25  * MA 02111-1307, USA.
  26  */
  27
  28 /*----- Header files ------------------------------------------------------*/
  29
  30 #include "config.h"
  31
  32 #include <assert.h>
  33 #include <ctype.h>
  34 #include <stdarg.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38
  39 #include <mLib/macros.h>
  40
  41 #include "dispatch.h"
  42
  43 /*----- Intel x86/AMD64 feature probing -----------------------------------*/
  44
  45 #if CPUFAM_X86 || CPUFAM_AMD64
  46
  47 enum {
  48   CPUID_1_D,                            /* eax = 1 => edx&?? */
  49 #  define CPUID1D_SSE2 (1u << 26)
  50 #  define CPUID1D_FXSR (1u << 24)
  51
  52   CPUID_1_C,                            /* eax = 1 => ecx&?? */
  53 #  define CPUID1C_PCLMUL (1u << 1)
  54 #  define CPUID1C_SSSE3 (1u << 9)
  55 #  define CPUID1C_AESNI (1u << 25)
  56 #  define CPUID1C_AVX (1u << 28)
  57 #  define CPUID1C_RDRAND (1u << 30)
  58
  59 };
  60
  61 struct cpuid { unsigned a, b, c, d; };
  62 extern int dispatch_x86ish_cpuid(struct cpuid *, unsigned a, unsigned c);
  63 extern int dispatch_x86ish_xmmregisters_p(void);
  64 extern int dispatch_x86ish_rdrand(unsigned *);
  65
  66 static void cpuid(struct cpuid *cc, unsigned a, unsigned c)
  67 {
  68   int rc = dispatch_x86ish_cpuid(cc, a, c);
  69   if (rc)
  70     dispatch_debug("CPUID instruction not available");
  71   else
  72     dispatch_debug("CPUID(%08x, %08x) -> %08x, %08x, %08x, %08x",
  73                    a, c, cc->a, cc->b, cc->c, cc->d);
  74 }
  75
  76 static unsigned cpuid_maxleaf(void)
  77   { struct cpuid c; cpuid(&c, 0, 0); return (c.a); }
  78
  79 /* --- @cpuid_feature_p@ --- *
  80  *
  81  * Arguments:   @unsigned leaf@ = leaf to look up
  82  *              @unsigned bits@ = bits to check
  83  *
  84  * Returns:     Nonzero if all the requested bits are set in the requested
  85  *              CPUID result.
  86  */
  87
  88 static int cpuid_feature_p(unsigned leaf, unsigned bits)
  89 {
  90   struct cpuid c;
  91   unsigned r;
  92
  93   switch (leaf) {
  94     case CPUID_1_D:
  95       if (cpuid_maxleaf() < 1) return (0);
  96       cpuid(&c, 1, 0); r = c.d;
  97       break;
  98     case CPUID_1_C:
  99       if (cpuid_maxleaf() < 1) return (0);
 100       cpuid(&c, 1, 0); r = c.c;
 101       break;
 102     default:
 103       assert(!"unknown cpuid leaf");
 104   }
 105   return ((r&bits) == bits);
 106 }
 107
 108 /* --- @xmm_registers_available_p@ --- *
 109  *
 110  * Arguments:   ---
 111  *
 112  * Returns:     Nonzero if the operating system has made the XMM registers
 113  *              available for use.
 114  */
 115
 116 static int xmm_registers_available_p(void)
 117 {
 118   int f = dispatch_x86ish_xmmregisters_p();
 119
 120   dispatch_debug("XMM registers %savailable", f ? "" : "not ");
 121   return (f);
 122 }
 123
 124 /* --- @rdrand_works_p@ --- *
 125  *
 126  *
 127  * Arguments:   ---
 128  *
 129  * Returns:     Nonzero if the `rdrand' instruction actually works.  Assumes
 130  *              that it's already been verified to be safe to issue.
 131  */
 132
 133 static int rdrand_works_p(void)
 134 {
 135   unsigned ref, x, i;
 136
 137   /* Check that it doesn't always give the same answer.  Try four times: this
 138    * will fail with probability %$2^{-128}$% with a truly random generator,
 139    * which seems fair enough.
 140    */
 141   if (dispatch_x86ish_rdrand(&ref)) goto fail;
 142   for (i = 0; i < 4; i++) {
 143     if (dispatch_x86ish_rdrand(&x)) goto fail;
 144     if (x != ref) goto not_stuck;
 145   }
 146   dispatch_debug("RDRAND always returns 0x%08x!", ref);
 147   return (0);
 148
 149 not_stuck:
 150   dispatch_debug("RDRAND instruction looks plausible");
 151   return (1);
 152
 153 fail:
 154   dispatch_debug("RDRAND instruction fails too often");
 155   return (0);
 156 }
 157
 158 #endif
 159
 160 /*----- General feature probing using auxiliary vectors -------------------*/
 161
 162 /* Try to find the system's definitions for auxiliary vector entries. */
 163 #ifdef HAVE_SYS_AUXV_H
 164 #  include <sys/auxv.h>
 165 #endif
 166 #ifdef HAVE_LINUX_AUXVEC_H
 167 #  include <linux/auxvec.h>
 168 #endif
 169 #ifdef HAVE_ASM_HWCAP_H
 170 #  include <asm/hwcap.h>
 171 #endif
 172
 173 /* The type of entries in the auxiliary vector.  I'm assuming that `unsigned
 174  * long' matches each platform's word length; if this is false then we'll
 175  * need some host-specific tweaking here.
 176  */
 177 union auxval { long i; unsigned long u; const void *p; };
 178 struct auxentry { unsigned long type; union auxval value; };
 179
 180 /* Register each CPU family's interest in the auxiliary vector.  Make sure
 181  * that the necessary entry types are defined.  This is primarily ordered by
 182  * entry type to minimize duplication.
 183  */
 184 #if defined(AT_HWCAP) && CPUFAM_ARMEL
 185 #  define WANT_ANY 1
 186 #  define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
 187 #endif
 188
 189 #if defined(AT_HWCAP) && CPUFAM_ARM64
 190 #  define WANT_ANY 1
 191 #  define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
 192 #endif
 193
 194 #if defined(AT_HWCAP2) && CPUFAM_ARMEL
 195 #  define WANT_ANY 1
 196 #  define WANT_AT_HWCAP2(_) _(AT_HWCAP2, u, hwcap2)
 197 #endif
 198
 199 /* If we couldn't find any interesting entries then we can switch all of this
 200  * machinery off.  Also do that if we have no means for atomic updates.
 201  */
 202 #if WANT_ANY && CPU_DISPATCH_P
 203
 204 /* The main output of this section is a bitmask of detected features.  The
 205  * least significant bit will be set if we've tried to probe.  Always access
 206  * this using `DISPATCH_LOAD' and `DISPATCH_STORE'.
 207  */
 208 static unsigned hwcaps = 0;
 209
 210 /* For each potentially interesting type which turned out not to exist or be
 211  * wanted, define a dummy macro for the sake of the next step.
 212  */
 213 #ifndef WANT_AT_HWCAP
 214 #  define WANT_AT_HWCAP(_)
 215 #endif
 216 #ifndef WANT_AT_HWCAP2
 217 #  define WANT_AT_HWCAP2(_)
 218 #endif
 219
 220 /* For each CPU family, define two lists.
 221  *
 222  *   * `WANTAUX' is a list of the `WANT_AT_MUMBLE' macros which the CPU
 223  *     family tried to register interest in above.  Each entry contains the
 224  *     interesting auxiliary vector entry type, the name of the union branch
 225  *     for its value, and the name of the slot in `struct auxprobe' in which
 226  *     to store the value.
 227  *
 228  *   * `CAPMAP' is a list describing the output features which the CPU family
 229  *     intends to satisfy from the auxiliary vector.  Each entry contains a
 230  *     feature name suffix, and the token name (for `check_env').
 231  */
 232 #if CPUFAM_ARMEL
 233 #  define WANTAUX(_)                                                    \
 234         WANT_AT_HWCAP(_)                                                \
 235         WANT_AT_HWCAP2(_)
 236 #  define CAPMAP(_)                                                     \
 237         _(ARM_VFP, "arm:vfp")                                           \
 238         _(ARM_NEON, "arm:neon")                                         \
 239         _(ARM_V4, "arm:v4")                                             \
 240         _(ARM_D32, "arm:d32")                                           \
 241         _(ARM_AES, "arm:aes")                                           \
 242         _(ARM_PMULL, "arm:pmull")
 243 #endif
 244 #if CPUFAM_ARM64
 245 #  define WANTAUX(_)                                                    \
 246         WANT_AT_HWCAP(_)
 247 #  define CAPMAP(_)                                                     \
 248         _(ARM_NEON, "arm:neon")                                         \
 249         _(ARM_AES, "arm:aes")                                           \
 250         _(ARM_PMULL, "arm:pmull")
 251 #endif
 252
 253 /* Build the bitmask for `hwcaps' from the `CAPMAP' list. */
 254 enum {
 255   HFI_PROBED = 0,
 256 #define HFI__ENUM(feat, tok) HFI_##feat,
 257   CAPMAP(HFI__ENUM)
 258 #undef HFI__ENUM
 259   HFI__END
 260 };
 261 enum {
 262   HF_PROBED = 1,
 263 #define HF__FLAG(feat, tok) HF_##feat = 1 << HFI_##feat,
 264   CAPMAP(HF__FLAG)
 265 #undef HF__FLAG
 266   HF__END
 267 };
 268
 269 /* Build a structure in which we can capture the interesting data from the
 270  * auxiliary vector.
 271  */
 272 #define AUXUTYPE_i long
 273 #define AUXUTYPE_u unsigned long
 274 #define AUXUTYPE_p const void *
 275 struct auxprobe {
 276 #define AUXPROBE__SLOT(type, ubranch, slot) AUXUTYPE_##ubranch slot;
 277   WANTAUX(AUXPROBE__SLOT)
 278 #undef AUXPROBE_SLOT
 279 };
 280
 281 /* --- @probe_hwcaps@ --- *
 282  *
 283  * Arguments:   ---
 284  *
 285  * Returns:     ---
 286  *
 287  * Use:         Attempt to find the auxiliary vector (which is well hidden)
 288  *              and discover interesting features from it.
 289  */
 290
 291 static void probe_hwcaps(void)
 292 {
 293   unsigned hw = HF_PROBED;
 294   struct auxprobe probed = { 0 };
 295
 296   /* Populate `probed' with the information we manage to retrieve from the
 297    * auxiliary vector.  Slots we couldn't find are left zero-valued.
 298    */
 299 #if defined(HAVE_GETAUXVAL)
 300   /* Shiny new libc lets us request individual entry types.  This is almost
 301    * too easy.
 302    */
 303 #  define CAP__GET(type, ubranch, slot)                                 \
 304         probed.slot = (AUXUTYPE_##ubranch)getauxval(type);
 305   WANTAUX(CAP__GET)
 306 #else
 307   /* Otherwise we're a bit stuck, really.  Modern Linux kernels make a copy
 308    * of the vector available in `/procc' so we could try that.
 309    *
 310    * The usual place is stuck on the end of the environment vector, but that
 311    * may well have moved, and we have no way of telling whether it has or
 312    * whether there was ever an auxiliary vector there at all; so don't do
 313    * that.
 314    */
 315   {
 316     FILE *fp = 0;
 317     unsigned char *p = 0, *q = 0;
 318     const struct auxentry *a;
 319     size_t sz, off, n;
 320
 321     /* Open the file and read it into a memory chunk. */
 322     if ((fp = fopen("/proc/self/auxv", "rb")) == 0) goto clean;
 323     sz = 4096; off = 0;
 324     if ((p = malloc(sz)) == 0) goto clean;
 325     for (;;) {
 326       n = fread(p + off, 1, sz - off, fp);
 327       off += n;
 328       if (off < sz) break;
 329       sz *= 2; if ((q = realloc(p, sz)) == 0) break;
 330       p = q;
 331     }
 332
 333     /* Work through the vector (or as much of it as we found) and extract the
 334      * types we're interested in.
 335      */
 336     for (a = (const struct auxentry *)p,
 337            n = sz/sizeof(struct auxentry);
 338          n--; a++) {
 339       switch (a->type) {
 340 #define CAP__SWITCH(type, ubranch, slot)                                \
 341         case type: probed.slot = a->value.ubranch; break;
 342         WANTAUX(CAP__SWITCH)
 343         case AT_NULL: goto clean;
 344       }
 345     }
 346
 347   clean:
 348     if (p) free(p);
 349     if (fp) fclose(fp);
 350   }
 351 #endif
 352
 353   /* Each CPU family now has to pick through what was found and stashed in
 354    * `probed', and set the appropriate flag bits in `hw'.
 355    */
 356 #if CPUFAM_ARMEL
 357   if (probed.hwcap & HWCAP_VFPv3) hw |= HF_ARM_VFP;
 358   if (probed.hwcap & HWCAP_NEON) hw |= HF_ARM_NEON;
 359   if (probed.hwcap & HWCAP_VFPD32) hw |= HF_ARM_D32;
 360   if (probed.hwcap & HWCAP_VFPv4) hw |= HF_ARM_V4;
 361 #  ifdef HWCAP2_AES
 362   if (probed.hwcap2 & HWCAP2_AES) hw |= HF_ARM_AES;
 363 #  endif
 364 #  ifdef HWCAP2_PMULL
 365   if (probed.hwcap2 & HWCAP2_PMULL) hw |= HF_ARM_PMULL;
 366 #  endif
 367 #endif
 368 #if CPUFAM_ARM64
 369   if (probed.hwcap & HWCAP_ASIMD) hw |= HF_ARM_NEON;
 370   if (probed.hwcap & HWCAP_AES) hw |= HF_ARM_AES;
 371   if (probed.hwcap & HWCAP_PMULL) hw |= HF_ARM_PMULL;
 372 #endif
 373
 374   /* Store the bitmask of features we probed for everyone to see. */
 375   DISPATCH_STORE(hwcaps, hw);
 376
 377   /* Finally, make a report about the things we found.  (Doing this earlier
 378    * will pointlessly widen the window in which multiple threads will do the
 379    * above auxiliary-vector probing.)
 380    */
 381 #define CAP__DEBUG(feat, tok)                                           \
 382   dispatch_debug("check auxv for feature `%s': %s", tok,                \
 383                  hw & HF_##feat ? "available" : "absent");
 384   CAPMAP(CAP__DEBUG)
 385 #undef CAP__DEBUG
 386 }
 387
 388 /* --- @get_hwcaps@ --- *
 389  *
 390  * Arguments:   ---
 391  *
 392  * Returns:     A mask of hardware capabilities and other features, as probed
 393  *              from the auxiliary vector.
 394  */
 395
 396 static unsigned get_hwcaps(void)
 397 {
 398   unsigned hw;
 399
 400   DISPATCH_LOAD(hwcaps, hw);
 401   if (!(hwcaps & HF_PROBED)) { probe_hwcaps(); DISPATCH_LOAD(hwcaps, hw); }
 402   return (hw);
 403 }
 404
 405 #endif
 406
 407 /*----- External interface ------------------------------------------------*/
 408
 409 /* --- @dispatch_debug@ --- *
 410  *
 411  * Arguments:   @const char *fmt@ = a format string
 412  *              @...@ = additional arguments
 413  *
 414  * Returns:     ---
 415  *
 416  * Use:         Writes a formatted message to standard output if dispatch
 417  *              debugging is enabled.
 418  */
 419
 420 void dispatch_debug(const char *fmt, ...)
 421 {
 422   va_list ap;
 423   const char *e = getenv("CATACOMB_CPUDISPATCH_DEBUG");
 424
 425   if (e && *e != 'n' && *e != '0') {
 426     va_start(ap, fmt);
 427     fputs("Catacomb CPUDISPATCH: ", stderr);
 428     vfprintf(stderr, fmt, ap);
 429     fputc('\n', stderr);
 430     va_end(ap);
 431   }
 432 }
 433
 434 /* --- @check_env@ --- *
 435  *
 436  * Arguments:   @const char *ftok@ = feature token
 437  *
 438  * Returns:     Zero if the feature is forced off; positive if it's forced
 439  *              on; negative if the user hasn't decided.
 440  *
 441  * Use:         Checks the environment variable `CATACOMB_CPUFEAT' for the
 442  *              feature token @ftok@.  The variable, if it exists, should be
 443  *              a space-separated sequence of `+tok' and `-tok' items.  These
 444  *              tokens may end in `*', which matches any suffix.
 445  */
 446
 447 static int IGNORABLE check_env(const char *ftok)
 448 {
 449   const char *p, *q, *pp;
 450   int d;
 451
 452   p = getenv("CATACOMB_CPUFEAT");
 453   if (!p) return (-1);
 454
 455   for (;;) {
 456     while (ISSPACE(*p)) p++;
 457     if (!*p) return (-1);
 458     switch (*p) {
 459       case '+': d = +1; p++; break;
 460       case '-': d =  0; p++; break;
 461       default:  d = -1;      break;
 462     }
 463     for (q = p; *q && !ISSPACE(*q); q++);
 464     if (d >= 0) {
 465       for (pp = ftok; p < q && *pp && *p == *pp; p++, pp++);
 466       if ((p == q && !*pp) || (*p == '*' && p + 1 == q)) return (d);
 467     }
 468     p = q;
 469   }
 470   return (-1);
 471 }
 472
 473 /* --- @cpu_feature_p@ --- *
 474  *
 475  * Arguments:   @unsigned feat@ = a @CPUFEAT_...@ code
 476  *
 477  * Returns:     Nonzero if the feature is available.
 478  */
 479
 480 #include <stdio.h>
 481
 482 static int IGNORABLE
 483   feat_debug(const char *ftok, const char *check, int verdict)
 484 {
 485   if (verdict >= 0) {
 486     dispatch_debug("feature `%s': %s -> %s", ftok, check,
 487                    verdict ? "available" : "absent");
 488   }
 489   return (verdict);
 490 }
 491
 492 int cpu_feature_p(int feat)
 493 {
 494   int IGNORABLE f;
 495   IGNORE(f);
 496 #define CASE_CPUFEAT(feat, ftok, cond) case CPUFEAT_##feat:             \
 497   if ((f = feat_debug(ftok, "environment override", check_env(ftok))) >= 0) \
 498     return (f);                                                         \
 499   else                                                                  \
 500     return (feat_debug(ftok, "runtime probe", cond));
 501
 502   switch (feat) {
 503 #if CPUFAM_X86 || CPUFAM_AMD64
 504     CASE_CPUFEAT(X86_SSE2, "x86:sse2",
 505                  cpuid_feature_p(CPUID_1_D, CPUID1D_SSE2) &&
 506                  xmm_registers_available_p());
 507     CASE_CPUFEAT(X86_AESNI, "x86:aesni",
 508                  cpuid_feature_p(CPUID_1_D, CPUID1C_AESNI) &&
 509                  xmm_registers_available_p());
 510     CASE_CPUFEAT(X86_RDRAND, "x86:rdrand",
 511                  cpuid_feature_p(CPUID_1_C, CPUID1C_RDRAND) &&
 512                  rdrand_works_p());
 513     CASE_CPUFEAT(X86_AVX, "x86:avx",
 514                  cpuid_feature_p(CPUID_1_C, CPUID1C_AVX) &&
 515                  xmm_registers_available_p());
 516     CASE_CPUFEAT(X86_SSSE3, "x86:ssse3",
 517                  cpuid_feature_p(CPUID_1_C, CPUID1C_SSSE3) &&
 518                  xmm_registers_available_p());
 519     CASE_CPUFEAT(X86_PCLMUL, "x86:pclmul",
 520                  cpuid_feature_p(CPUID_1_C, CPUID1C_PCLMUL) &&
 521                  xmm_registers_available_p());
 522 #endif
 523 #ifdef CAPMAP
 524 #  define FEATP__CASE(feat, tok)                                        \
 525         CASE_CPUFEAT(feat, tok, get_hwcaps() & HF_##feat)
 526     CAPMAP(FEATP__CASE)
 527 #undef FEATP__CASE
 528 #endif
 529     default:
 530       dispatch_debug("denying unknown feature %d", feat);
 531       return (0);
 532   }
 533 #undef CASE_CPUFEAT
 534 }
 535
 536 /*----- That's all, folks -------------------------------------------------*/