mdw@git.distorted.org.uk Git - catacomb/blob - base/dispatch.c

   1 /* -*-c-*-
   2  *
   3  * CPU-specific dispatch
   4  *
   5  * (c) 2015 Straylight/Edgeware
   6  */
   7
   8 /*----- Licensing notice --------------------------------------------------*
   9  *
  10  * This file is part of Catacomb.
  11  *
  12  * Catacomb is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU Library General Public License as
  14  * published by the Free Software Foundation; either version 2 of the
  15  * License, or (at your option) any later version.
  16  *
  17  * Catacomb is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20  * GNU Library General Public License for more details.
  21  *
  22  * You should have received a copy of the GNU Library General Public
  23  * License along with Catacomb; if not, write to the Free
  24  * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
  25  * MA 02111-1307, USA.
  26  */
  27
  28 /*----- Header files ------------------------------------------------------*/
  29
  30 #include "config.h"
  31
  32 #include <assert.h>
  33 #include <ctype.h>
  34 #include <stdarg.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38
  39 #include <mLib/macros.h>
  40
  41 #include "dispatch.h"
  42
  43 /*----- Intel x86/AMD64 feature probing -----------------------------------*/
  44
  45 #if CPUFAM_X86 || CPUFAM_AMD64
  46
  47 enum {
  48   CPUID_1_D,                            /* eax = 1 => edx&?? */
  49 #  define CPUID1D_SSE2 (1u << 26)
  50 #  define CPUID1D_FXSR (1u << 24)
  51
  52   CPUID_1_C,                            /* eax = 1 => ecx&?? */
  53 #  define CPUID1C_PCLMUL (1u << 1)
  54 #  define CPUID1C_SSSE3 (1u << 9)
  55 #  define CPUID1C_AESNI (1u << 25)
  56 #  define CPUID1C_OSXSAVE (1u << 27)
  57 #  define CPUID1C_AVX (1u << 28)
  58 #  define CPUID1C_RDRAND (1u << 30)
  59
  60   CPUID_7_0_B,                          /* eax = 7, ecx = 0 => ebx&?? */
  61 #  define CPUID70B_AVX2 (1u << 5)
  62 #  define CPUID70B_RDSEED (1u << 18)
  63 };
  64
  65 struct cpuid { unsigned a, b, c, d; };
  66 struct xcr { unsigned lo, hi; };
  67 extern int dispatch_x86ish_cpuid(struct cpuid *, unsigned a, unsigned c);
  68 extern int dispatch_x86ish_xmmregisters_p(void);
  69 extern int dispatch_x86ish_xgetbv(struct xcr *z_out, unsigned c);
  70 extern int dispatch_x86ish_rdrand(unsigned op, unsigned *);
  71
  72 static void cpuid(struct cpuid *cc, unsigned a, unsigned c)
  73 {
  74   int rc = dispatch_x86ish_cpuid(cc, a, c);
  75   if (rc)
  76     dispatch_debug("CPUID instruction not available");
  77   else
  78     dispatch_debug("CPUID(%08x, %08x) -> %08x, %08x, %08x, %08x",
  79                    a, c, cc->a, cc->b, cc->c, cc->d);
  80 }
  81
  82 static unsigned cpuid_maxleaf(void)
  83   { struct cpuid c; cpuid(&c, 0, 0); return (c.a); }
  84
  85 /* --- @cpuid_feature_p@ --- *
  86  *
  87  * Arguments:   @unsigned leaf@ = leaf to look up
  88  *              @unsigned bits@ = bits to check
  89  *
  90  * Returns:     Nonzero if all the requested bits are set in the requested
  91  *              CPUID result.
  92  */
  93
  94 static int cpuid_feature_p(unsigned leaf, unsigned bits)
  95 {
  96   struct cpuid c;
  97   unsigned r;
  98
  99   switch (leaf) {
 100     case CPUID_1_D:
 101       if (cpuid_maxleaf() < 1) return (0);
 102       cpuid(&c, 1, 0); r = c.d;
 103       break;
 104     case CPUID_1_C:
 105       if (cpuid_maxleaf() < 1) return (0);
 106       cpuid(&c, 1, 0); r = c.c;
 107       break;
 108     case CPUID_7_0_B:
 109       if (cpuid_maxleaf() < 7) return (0);
 110       cpuid(&c, 7, 0); r = c.b;
 111       break;
 112     default:
 113       assert(!"unknown cpuid leaf");
 114   }
 115   return ((r&bits) == bits);
 116 }
 117
 118 /* --- @{x,y}mm_registers_available_p@ --- *
 119  *
 120  * Arguments:   ---
 121  *
 122  * Returns:     Nonzero if the operating system has made the XMM or YMM
 123  *              registers available for use.
 124  */
 125
 126 static int xmm_registers_available_p(void)
 127 {
 128   int f = dispatch_x86ish_xmmregisters_p();
 129
 130   dispatch_debug("XMM registers %savailable", f ? "" : "not ");
 131   return (f);
 132 }
 133
 134 static int ymm_registers_available_p(void)
 135 {
 136   struct xcr xcr0;
 137   int f;
 138
 139   f = cpuid_feature_p(CPUID_1_C, CPUID1C_OSXSAVE);
 140   dispatch_debug("XGETBV %savailable", f ? "" : "not ");
 141   if (!f) return (0);
 142
 143   dispatch_x86ish_xgetbv(&xcr0, 0); f = (xcr0.lo&0x06) == 0x06;
 144   dispatch_debug("YMM state %senabled", f ? "" : "not ");
 145   if (!f) return (0);
 146
 147   return (1);
 148 }
 149
 150 /* --- @rdrand_works_p@ --- *
 151  *
 152  *
 153  * Arguments:   ---
 154  *
 155  * Returns:     Nonzero if the `rdrand' instruction actually works.  Assumes
 156  *              that it's already been verified to be safe to issue.
 157  */
 158
 159 enum { OP_RDRAND, OP_RDSEED };
 160
 161 static int rdrand_works_p(unsigned op)
 162 {
 163   unsigned ref, x, i;
 164   const char *what;
 165
 166   switch (op) {
 167     case OP_RDRAND: what = "RDRAND"; break;
 168     case OP_RDSEED: what = "RDSEED"; break;
 169     default: assert(!"unexpected op");
 170   }
 171
 172   /* Check that it doesn't always give the same answer.  Try four times: this
 173    * will fail with probability %$2^{-128}$% with a truly random generator,
 174    * which seems fair enough.
 175    */
 176   if (dispatch_x86ish_rdrand(op, &ref)) goto fail;
 177   for (i = 0; i < 4; i++) {
 178     if (dispatch_x86ish_rdrand(op, &x)) goto fail;
 179     if (x != ref) goto not_stuck;
 180   }
 181   dispatch_debug("%s always returns 0x%08x!", what, ref);
 182   return (0);
 183
 184 not_stuck:
 185   dispatch_debug("%s instruction looks plausible", what);
 186   return (1);
 187
 188 fail:
 189   dispatch_debug("%s instruction fails too often", what);
 190   return (0);
 191 }
 192
 193 #endif
 194
 195 /*----- General feature probing using auxiliary vectors -------------------*/
 196
 197 /* Try to find the system's definitions for auxiliary vector entries. */
 198 #ifdef HAVE_SYS_AUXV_H
 199 #  include <sys/auxv.h>
 200 #endif
 201 #ifdef HAVE_LINUX_AUXVEC_H
 202 #  include <linux/auxvec.h>
 203 #endif
 204 #ifdef HAVE_ASM_HWCAP_H
 205 #  include <asm/hwcap.h>
 206 #endif
 207
 208 /* The type of entries in the auxiliary vector.  I'm assuming that `unsigned
 209  * long' matches each platform's word length; if this is false then we'll
 210  * need some host-specific tweaking here.
 211  */
 212 union auxval { long i; unsigned long u; const void *p; };
 213 struct auxentry { unsigned long type; union auxval value; };
 214
 215 /* Register each CPU family's interest in the auxiliary vector.  Make sure
 216  * that the necessary entry types are defined.  This is primarily ordered by
 217  * entry type to minimize duplication.
 218  */
 219 #if defined(AT_HWCAP) && CPUFAM_ARMEL
 220 #  define WANT_ANY 1
 221 #  define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
 222 #endif
 223
 224 #if defined(AT_HWCAP) && CPUFAM_ARM64
 225 #  define WANT_ANY 1
 226 #  define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
 227 #endif
 228
 229 #if defined(AT_HWCAP2) && CPUFAM_ARMEL
 230 #  define WANT_ANY 1
 231 #  define WANT_AT_HWCAP2(_) _(AT_HWCAP2, u, hwcap2)
 232 #endif
 233
 234 /* If we couldn't find any interesting entries then we can switch all of this
 235  * machinery off.  Also do that if we have no means for atomic updates.
 236  */
 237 #if WANT_ANY && CPU_DISPATCH_P
 238
 239 /* The main output of this section is a bitmask of detected features.  The
 240  * least significant bit will be set if we've tried to probe.  Always access
 241  * this using `DISPATCH_LOAD' and `DISPATCH_STORE'.
 242  */
 243 static unsigned hwcaps = 0;
 244
 245 /* For each potentially interesting type which turned out not to exist or be
 246  * wanted, define a dummy macro for the sake of the next step.
 247  */
 248 #ifndef WANT_AT_HWCAP
 249 #  define WANT_AT_HWCAP(_)
 250 #endif
 251 #ifndef WANT_AT_HWCAP2
 252 #  define WANT_AT_HWCAP2(_)
 253 #endif
 254
 255 /* For each CPU family, define two lists.
 256  *
 257  *   * `WANTAUX' is a list of the `WANT_AT_MUMBLE' macros which the CPU
 258  *     family tried to register interest in above.  Each entry contains the
 259  *     interesting auxiliary vector entry type, the name of the union branch
 260  *     for its value, and the name of the slot in `struct auxprobe' in which
 261  *     to store the value.
 262  *
 263  *   * `CAPMAP' is a list describing the output features which the CPU family
 264  *     intends to satisfy from the auxiliary vector.  Each entry contains a
 265  *     feature name suffix, and the token name (for `check_env').
 266  */
 267 #if CPUFAM_ARMEL
 268 #  define WANTAUX(_)                                                    \
 269         WANT_AT_HWCAP(_)                                                \
 270         WANT_AT_HWCAP2(_)
 271 #  define CAPMAP(_)                                                     \
 272         _(ARM_VFP, "arm:vfp")                                           \
 273         _(ARM_NEON, "arm:neon")                                         \
 274         _(ARM_V4, "arm:v4")                                             \
 275         _(ARM_D32, "arm:d32")                                           \
 276         _(ARM_AES, "arm:aes")                                           \
 277         _(ARM_PMULL, "arm:pmull")
 278 #endif
 279 #if CPUFAM_ARM64
 280 #  define WANTAUX(_)                                                    \
 281         WANT_AT_HWCAP(_)
 282 #  define CAPMAP(_)                                                     \
 283         _(ARM_NEON, "arm:neon")                                         \
 284         _(ARM_AES, "arm:aes")                                           \
 285         _(ARM_PMULL, "arm:pmull")
 286 #endif
 287
 288 /* Build the bitmask for `hwcaps' from the `CAPMAP' list. */
 289 enum {
 290   HFI_PROBED = 0,
 291 #define HFI__ENUM(feat, tok) HFI_##feat,
 292   CAPMAP(HFI__ENUM)
 293 #undef HFI__ENUM
 294   HFI__END
 295 };
 296 enum {
 297   HF_PROBED = 1,
 298 #define HF__FLAG(feat, tok) HF_##feat = 1 << HFI_##feat,
 299   CAPMAP(HF__FLAG)
 300 #undef HF__FLAG
 301   HF__END
 302 };
 303
 304 /* Build a structure in which we can capture the interesting data from the
 305  * auxiliary vector.
 306  */
 307 #define AUXUTYPE_i long
 308 #define AUXUTYPE_u unsigned long
 309 #define AUXUTYPE_p const void *
 310 struct auxprobe {
 311 #define AUXPROBE__SLOT(type, ubranch, slot) AUXUTYPE_##ubranch slot;
 312   WANTAUX(AUXPROBE__SLOT)
 313 #undef AUXPROBE_SLOT
 314 };
 315
 316 /* --- @probe_hwcaps@ --- *
 317  *
 318  * Arguments:   ---
 319  *
 320  * Returns:     ---
 321  *
 322  * Use:         Attempt to find the auxiliary vector (which is well hidden)
 323  *              and discover interesting features from it.
 324  */
 325
 326 static void probe_hwcaps(void)
 327 {
 328   unsigned hw = HF_PROBED;
 329   struct auxprobe probed = { 0 };
 330
 331   /* Populate `probed' with the information we manage to retrieve from the
 332    * auxiliary vector.  Slots we couldn't find are left zero-valued.
 333    */
 334 #if defined(HAVE_GETAUXVAL)
 335   /* Shiny new libc lets us request individual entry types.  This is almost
 336    * too easy.
 337    */
 338 #  define CAP__GET(type, ubranch, slot)                                 \
 339         probed.slot = (AUXUTYPE_##ubranch)getauxval(type);
 340   WANTAUX(CAP__GET)
 341 #else
 342   /* Otherwise we're a bit stuck, really.  Modern Linux kernels make a copy
 343    * of the vector available in `/procc' so we could try that.
 344    *
 345    * The usual place is stuck on the end of the environment vector, but that
 346    * may well have moved, and we have no way of telling whether it has or
 347    * whether there was ever an auxiliary vector there at all; so don't do
 348    * that.
 349    */
 350   {
 351     FILE *fp = 0;
 352     unsigned char *p = 0, *q = 0;
 353     const struct auxentry *a;
 354     size_t sz, off, n;
 355
 356     /* Open the file and read it into a memory chunk. */
 357     if ((fp = fopen("/proc/self/auxv", "rb")) == 0) goto clean;
 358     sz = 4096; off = 0;
 359     if ((p = malloc(sz)) == 0) goto clean;
 360     for (;;) {
 361       n = fread(p + off, 1, sz - off, fp);
 362       off += n;
 363       if (off < sz) break;
 364       sz *= 2; if ((q = realloc(p, sz)) == 0) break;
 365       p = q;
 366     }
 367
 368     /* Work through the vector (or as much of it as we found) and extract the
 369      * types we're interested in.
 370      */
 371     for (a = (const struct auxentry *)p,
 372            n = sz/sizeof(struct auxentry);
 373          n--; a++) {
 374       switch (a->type) {
 375 #define CAP__SWITCH(type, ubranch, slot)                                \
 376         case type: probed.slot = a->value.ubranch; break;
 377         WANTAUX(CAP__SWITCH)
 378         case AT_NULL: goto clean;
 379       }
 380     }
 381
 382   clean:
 383     if (p) free(p);
 384     if (fp) fclose(fp);
 385   }
 386 #endif
 387
 388   /* Each CPU family now has to pick through what was found and stashed in
 389    * `probed', and set the appropriate flag bits in `hw'.
 390    */
 391 #if CPUFAM_ARMEL
 392   if (probed.hwcap & HWCAP_VFPv3) hw |= HF_ARM_VFP;
 393   if (probed.hwcap & HWCAP_NEON) hw |= HF_ARM_NEON;
 394   if (probed.hwcap & HWCAP_VFPD32) hw |= HF_ARM_D32;
 395   if (probed.hwcap & HWCAP_VFPv4) hw |= HF_ARM_V4;
 396 #  ifdef HWCAP2_AES
 397   if (probed.hwcap2 & HWCAP2_AES) hw |= HF_ARM_AES;
 398 #  endif
 399 #  ifdef HWCAP2_PMULL
 400   if (probed.hwcap2 & HWCAP2_PMULL) hw |= HF_ARM_PMULL;
 401 #  endif
 402 #endif
 403 #if CPUFAM_ARM64
 404   if (probed.hwcap & HWCAP_ASIMD) hw |= HF_ARM_NEON;
 405   if (probed.hwcap & HWCAP_AES) hw |= HF_ARM_AES;
 406   if (probed.hwcap & HWCAP_PMULL) hw |= HF_ARM_PMULL;
 407 #endif
 408
 409   /* Store the bitmask of features we probed for everyone to see. */
 410   DISPATCH_STORE(hwcaps, hw);
 411
 412   /* Finally, make a report about the things we found.  (Doing this earlier
 413    * will pointlessly widen the window in which multiple threads will do the
 414    * above auxiliary-vector probing.)
 415    */
 416 #define CAP__DEBUG(feat, tok)                                           \
 417   dispatch_debug("check auxv for feature `%s': %s", tok,                \
 418                  hw & HF_##feat ? "available" : "absent");
 419   CAPMAP(CAP__DEBUG)
 420 #undef CAP__DEBUG
 421 }
 422
 423 /* --- @get_hwcaps@ --- *
 424  *
 425  * Arguments:   ---
 426  *
 427  * Returns:     A mask of hardware capabilities and other features, as probed
 428  *              from the auxiliary vector.
 429  */
 430
 431 static unsigned get_hwcaps(void)
 432 {
 433   unsigned hw;
 434
 435   DISPATCH_LOAD(hwcaps, hw);
 436   if (!(hw & HF_PROBED)) { probe_hwcaps(); DISPATCH_LOAD(hwcaps, hw); }
 437   return (hw);
 438 }
 439
 440 #endif
 441
 442 /*----- External interface ------------------------------------------------*/
 443
 444 /* --- @dispatch_debug@ --- *
 445  *
 446  * Arguments:   @const char *fmt@ = a format string
 447  *              @...@ = additional arguments
 448  *
 449  * Returns:     ---
 450  *
 451  * Use:         Writes a formatted message to standard output if dispatch
 452  *              debugging is enabled.
 453  */
 454
 455 void dispatch_debug(const char *fmt, ...)
 456 {
 457   va_list ap;
 458   const char *e = getenv("CATACOMB_CPUDISPATCH_DEBUG");
 459
 460   if (e && *e != 'n' && *e != '0') {
 461     va_start(ap, fmt);
 462     fputs("Catacomb CPUDISPATCH: ", stderr);
 463     vfprintf(stderr, fmt, ap);
 464     fputc('\n', stderr);
 465     va_end(ap);
 466   }
 467 }
 468
 469 /* --- @check_env@ --- *
 470  *
 471  * Arguments:   @const char *ftok@ = feature token
 472  *
 473  * Returns:     Zero if the feature is forced off; positive if it's forced
 474  *              on; negative if the user hasn't decided.
 475  *
 476  * Use:         Checks the environment variable `CATACOMB_CPUFEAT' for the
 477  *              feature token @ftok@.  The variable, if it exists, should be
 478  *              a space-separated sequence of `+tok' and `-tok' items.  These
 479  *              tokens may end in `*', which matches any suffix.
 480  */
 481
 482 static int IGNORABLE check_env(const char *ftok)
 483 {
 484   const char *p, *q, *pp;
 485   int d;
 486
 487   p = getenv("CATACOMB_CPUFEAT");
 488   if (!p) return (-1);
 489
 490   for (;;) {
 491     while (ISSPACE(*p)) p++;
 492     if (!*p) return (-1);
 493     switch (*p) {
 494       case '+': d = +1; p++; break;
 495       case '-': d =  0; p++; break;
 496       default:  d = -1;      break;
 497     }
 498     for (q = p; *q && !ISSPACE(*q); q++);
 499     if (d >= 0) {
 500       for (pp = ftok; p < q && *pp && *p == *pp; p++, pp++);
 501       if ((p == q && !*pp) || (*p == '*' && p + 1 == q)) return (d);
 502     }
 503     p = q;
 504   }
 505   return (-1);
 506 }
 507
 508 /* --- @cpu_feature_p@ --- *
 509  *
 510  * Arguments:   @unsigned feat@ = a @CPUFEAT_...@ code
 511  *
 512  * Returns:     Nonzero if the feature is available.
 513  */
 514
 515 #include <stdio.h>
 516
 517 static int IGNORABLE
 518   feat_debug(const char *ftok, const char *check, int verdict)
 519 {
 520   if (verdict >= 0) {
 521     dispatch_debug("feature `%s': %s -> %s", ftok, check,
 522                    verdict ? "available" : "absent");
 523   }
 524   return (verdict);
 525 }
 526
 527 int cpu_feature_p(int feat)
 528 {
 529   int IGNORABLE f;
 530   IGNORE(f);
 531 #define CASE_CPUFEAT(feat, ftok, cond) case CPUFEAT_##feat:             \
 532   if ((f = feat_debug(ftok, "environment override", check_env(ftok))) >= 0) \
 533     return (f);                                                         \
 534   else                                                                  \
 535     return (feat_debug(ftok, "runtime probe", cond));
 536
 537   switch (feat) {
 538 #if CPUFAM_X86 || CPUFAM_AMD64
 539     CASE_CPUFEAT(X86_SSE2, "x86:sse2",
 540                  cpuid_feature_p(CPUID_1_D, CPUID1D_SSE2) &&
 541                  xmm_registers_available_p());
 542     CASE_CPUFEAT(X86_AESNI, "x86:aesni",
 543                  cpuid_feature_p(CPUID_1_C, CPUID1C_AESNI) &&
 544                  xmm_registers_available_p());
 545     CASE_CPUFEAT(X86_RDRAND, "x86:rdrand",
 546                  cpuid_feature_p(CPUID_1_C, CPUID1C_RDRAND) &&
 547                  rdrand_works_p(OP_RDRAND));
 548     CASE_CPUFEAT(X86_AVX, "x86:avx",
 549                  cpuid_feature_p(CPUID_1_C, CPUID1C_AVX) &&
 550                  ymm_registers_available_p());
 551     CASE_CPUFEAT(X86_AVX2, "x86:avx2",
 552                  cpuid_feature_p(CPUID_1_C, CPUID1C_AVX) &&
 553                  cpuid_feature_p(CPUID_7_0_B, CPUID70B_AVX2) &&
 554                  ymm_registers_available_p());
 555     CASE_CPUFEAT(X86_SSSE3, "x86:ssse3",
 556                  cpuid_feature_p(CPUID_1_C, CPUID1C_SSSE3) &&
 557                  xmm_registers_available_p());
 558     CASE_CPUFEAT(X86_PCLMUL, "x86:pclmul",
 559                  cpuid_feature_p(CPUID_1_C, CPUID1C_PCLMUL) &&
 560                  xmm_registers_available_p());
 561     CASE_CPUFEAT(X86_RDSEED, "x86:rdseed",
 562                  cpuid_feature_p(CPUID_7_0_B, CPUID70B_RDSEED) &&
 563                  rdrand_works_p(OP_RDSEED));
 564 #endif
 565 #ifdef CAPMAP
 566 #  define FEATP__CASE(feat, tok)                                        \
 567         CASE_CPUFEAT(feat, tok, get_hwcaps() & HF_##feat)
 568     CAPMAP(FEATP__CASE)
 569 #undef FEATP__CASE
 570 #endif
 571     default:
 572       dispatch_debug("denying unknown feature %d", feat);
 573       return (0);
 574   }
 575 #undef CASE_CPUFEAT
 576 }
 577
 578 /*----- That's all, folks -------------------------------------------------*/