3 * CPU-specific dispatch
5 * (c) 2015 Straylight/Edgeware
8 /*----- Licensing notice --------------------------------------------------*
10 * This file is part of Catacomb.
12 * Catacomb is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU Library General Public License as
14 * published by the Free Software Foundation; either version 2 of the
15 * License, or (at your option) any later version.
17 * Catacomb is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU Library General Public License for more details.
22 * You should have received a copy of the GNU Library General Public
23 * License along with Catacomb; if not, write to the Free
24 * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
28 /*----- Header files ------------------------------------------------------*/
39 #include <mLib/macros.h>
43 /*----- Intel x86/AMD64 feature probing -----------------------------------*/
45 #if CPUFAM_X86 || CPUFAM_AMD64
48 CPUID_1_D
, /* eax = 1 => edx&?? */
49 # define CPUID1D_SSE2 (1u << 26)
50 # define CPUID1D_FXSR (1u << 24)
52 CPUID_1_C
, /* eax = 1 => ecx&?? */
53 # define CPUID1C_PCLMUL (1u << 1)
54 # define CPUID1C_SSSE3 (1u << 9)
55 # define CPUID1C_AESNI (1u << 25)
56 # define CPUID1C_OSXSAVE (1u << 27)
57 # define CPUID1C_AVX (1u << 28)
58 # define CPUID1C_RDRAND (1u << 30)
60 CPUID_7_0_B
, /* eax = 7, ecx = 0 => ebx&?? */
61 # define CPUID70B_AVX2 (1u << 5)
62 # define CPUID70B_RDSEED (1u << 18)
65 struct cpuid
{ unsigned a
, b
, c
, d
; };
66 struct xcr
{ unsigned lo
, hi
; };
67 extern int dispatch_x86ish_cpuid(struct cpuid
*, unsigned a
, unsigned c
);
68 extern int dispatch_x86ish_xmmregisters_p(void);
69 extern int dispatch_x86ish_xgetbv(struct xcr
*z_out
, unsigned c
);
70 extern int dispatch_x86ish_rdrand(unsigned op
, unsigned *);
72 static void cpuid(struct cpuid
*cc
, unsigned a
, unsigned c
)
74 int rc
= dispatch_x86ish_cpuid(cc
, a
, c
);
76 dispatch_debug("CPUID instruction not available");
78 dispatch_debug("CPUID(%08x, %08x) -> %08x, %08x, %08x, %08x",
79 a
, c
, cc
->a
, cc
->b
, cc
->c
, cc
->d
);
82 static unsigned cpuid_maxleaf(void)
83 { struct cpuid c
; cpuid(&c
, 0, 0); return (c
.a
); }
85 /* --- @cpuid_feature_p@ --- *
87 * Arguments: @unsigned leaf@ = leaf to look up
88 * @unsigned bits@ = bits to check
90 * Returns: Nonzero if all the requested bits are set in the requested
94 static int cpuid_feature_p(unsigned leaf
, unsigned bits
)
101 if (cpuid_maxleaf() < 1) return (0);
102 cpuid(&c
, 1, 0); r
= c
.d
;
105 if (cpuid_maxleaf() < 1) return (0);
106 cpuid(&c
, 1, 0); r
= c
.c
;
109 if (cpuid_maxleaf() < 7) return (0);
110 cpuid(&c
, 7, 0); r
= c
.b
;
113 assert(!"unknown cpuid leaf");
115 return ((r
&bits
) == bits
);
118 /* --- @{x,y}mm_registers_available_p@ --- *
122 * Returns: Nonzero if the operating system has made the XMM or YMM
123 * registers available for use.
126 static int xmm_registers_available_p(void)
128 int f
= dispatch_x86ish_xmmregisters_p();
130 dispatch_debug("XMM registers %savailable", f ?
"" : "not ");
134 static int ymm_registers_available_p(void)
139 f
= cpuid_feature_p(CPUID_1_C
, CPUID1C_OSXSAVE
);
140 dispatch_debug("XGETBV %savailable", f ?
"" : "not ");
143 dispatch_x86ish_xgetbv(&xcr0
, 0); f
= (xcr0
.lo
&0x06) == 0x06;
144 dispatch_debug("YMM state %senabled", f ?
"" : "not ");
150 /* --- @rdrand_works_p@ --- *
155 * Returns: Nonzero if the `rdrand' instruction actually works. Assumes
156 * that it's already been verified to be safe to issue.
159 enum { OP_RDRAND
, OP_RDSEED
};
161 static int rdrand_works_p(unsigned op
)
167 case OP_RDRAND
: what
= "RDRAND"; break;
168 case OP_RDSEED
: what
= "RDSEED"; break;
169 default: assert(!"unexpected op");
172 /* Check that it doesn't always give the same answer. Try four times: this
173 * will fail with probability %$2^{-128}$% with a truly random generator,
174 * which seems fair enough.
176 if (dispatch_x86ish_rdrand(op
, &ref
)) goto fail
;
177 for (i
= 0; i
< 4; i
++) {
178 if (dispatch_x86ish_rdrand(op
, &x
)) goto fail
;
179 if (x
!= ref
) goto not_stuck
;
181 dispatch_debug("%s always returns 0x%08x!", what
, ref
);
185 dispatch_debug("%s instruction looks plausible", what
);
189 dispatch_debug("%s instruction fails too often", what
);
195 /*----- General feature probing using auxiliary vectors -------------------*/
197 /* Try to find the system's definitions for auxiliary vector entries. */
198 #ifdef HAVE_SYS_AUXV_H
199 # include <sys/auxv.h>
201 #ifdef HAVE_LINUX_AUXVEC_H
202 # include <linux/auxvec.h>
204 #ifdef HAVE_ASM_HWCAP_H
205 # include <asm/hwcap.h>
208 /* The type of entries in the auxiliary vector. I'm assuming that `unsigned
209 * long' matches each platform's word length; if this is false then we'll
210 * need some host-specific tweaking here.
212 union auxval
{ long i
; unsigned long u
; const void *p
; };
213 struct auxentry
{ unsigned long type
; union auxval value
; };
215 /* Register each CPU family's interest in the auxiliary vector. Make sure
216 * that the necessary entry types are defined. This is primarily ordered by
217 * entry type to minimize duplication.
219 #if defined(AT_HWCAP) && CPUFAM_ARMEL
221 # define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
224 #if defined(AT_HWCAP) && CPUFAM_ARM64
226 # define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
229 #if defined(AT_HWCAP2) && CPUFAM_ARMEL
231 # define WANT_AT_HWCAP2(_) _(AT_HWCAP2, u, hwcap2)
234 /* If we couldn't find any interesting entries then we can switch all of this
235 * machinery off. Also do that if we have no means for atomic updates.
237 #if WANT_ANY && CPU_DISPATCH_P
239 /* The main output of this section is a bitmask of detected features. The
240 * least significant bit will be set if we've tried to probe. Always access
241 * this using `DISPATCH_LOAD' and `DISPATCH_STORE'.
243 static unsigned hwcaps
= 0;
245 /* For each potentially interesting type which turned out not to exist or be
246 * wanted, define a dummy macro for the sake of the next step.
248 #ifndef WANT_AT_HWCAP
249 # define WANT_AT_HWCAP(_)
251 #ifndef WANT_AT_HWCAP2
252 # define WANT_AT_HWCAP2(_)
255 /* For each CPU family, define two lists.
257 * * `WANTAUX' is a list of the `WANT_AT_MUMBLE' macros which the CPU
258 * family tried to register interest in above. Each entry contains the
259 * interesting auxiliary vector entry type, the name of the union branch
260 * for its value, and the name of the slot in `struct auxprobe' in which
261 * to store the value.
263 * * `CAPMAP' is a list describing the output features which the CPU family
264 * intends to satisfy from the auxiliary vector. Each entry contains a
265 * feature name suffix, and the token name (for `check_env').
268 # define WANTAUX(_) \
272 _(ARM_VFP, "arm:vfp") \
273 _(ARM_NEON, "arm:neon") \
274 _(ARM_V4, "arm:v4") \
275 _(ARM_D32, "arm:d32") \
276 _(ARM_AES, "arm:aes") \
277 _(ARM_PMULL, "arm:pmull")
280 # define WANTAUX(_) \
283 _(ARM_NEON, "arm:neon") \
284 _(ARM_AES, "arm:aes") \
285 _(ARM_PMULL, "arm:pmull")
288 /* Build the bitmask for `hwcaps' from the `CAPMAP' list. */
291 #define HFI__ENUM(feat, tok) HFI_##feat,
298 #define HF__FLAG(feat, tok) HF_##feat = 1 << HFI_##feat,
304 /* Build a structure in which we can capture the interesting data from the
307 #define AUXUTYPE_i long
308 #define AUXUTYPE_u unsigned long
309 #define AUXUTYPE_p const void *
311 #define AUXPROBE__SLOT(type, ubranch, slot) AUXUTYPE_##ubranch slot;
312 WANTAUX(AUXPROBE__SLOT
)
316 /* --- @probe_hwcaps@ --- *
322 * Use: Attempt to find the auxiliary vector (which is well hidden)
323 * and discover interesting features from it.
326 static void probe_hwcaps(void)
328 unsigned hw
= HF_PROBED
;
329 struct auxprobe probed
= { 0 };
331 /* Populate `probed' with the information we manage to retrieve from the
332 * auxiliary vector. Slots we couldn't find are left zero-valued.
334 #if defined(HAVE_GETAUXVAL)
335 /* Shiny new libc lets us request individual entry types. This is almost
338 # define CAP__GET(type, ubranch, slot) \
339 probed.slot = (AUXUTYPE_##ubranch)getauxval(type);
342 /* Otherwise we're a bit stuck, really. Modern Linux kernels make a copy
343 * of the vector available in `/procc' so we could try that.
345 * The usual place is stuck on the end of the environment vector, but that
346 * may well have moved, and we have no way of telling whether it has or
347 * whether there was ever an auxiliary vector there at all; so don't do
352 unsigned char *p
= 0, *q
= 0;
353 const struct auxentry
*a
;
356 /* Open the file and read it into a memory chunk. */
357 if ((fp
= fopen("/proc/self/auxv", "rb")) == 0) goto clean
;
359 if ((p
= malloc(sz
)) == 0) goto clean
;
361 n
= fread(p
+ off
, 1, sz
- off
, fp
);
364 sz
*= 2; if ((q
= realloc(p
, sz
)) == 0) break;
368 /* Work through the vector (or as much of it as we found) and extract the
369 * types we're interested in.
371 for (a
= (const struct auxentry
*)p
,
372 n
= sz
/sizeof(struct auxentry
);
375 #define CAP__SWITCH(type, ubranch, slot) \
376 case type: probed.slot = a->value.ubranch; break;
378 case AT_NULL
: goto clean
;
388 /* Each CPU family now has to pick through what was found and stashed in
389 * `probed', and set the appropriate flag bits in `hw'.
392 if (probed
.hwcap
& HWCAP_VFPv3
) hw
|= HF_ARM_VFP
;
393 if (probed
.hwcap
& HWCAP_NEON
) hw
|= HF_ARM_NEON
;
394 if (probed
.hwcap
& HWCAP_VFPD32
) hw
|= HF_ARM_D32
;
395 if (probed
.hwcap
& HWCAP_VFPv4
) hw
|= HF_ARM_V4
;
397 if (probed
.hwcap2
& HWCAP2_AES
) hw
|= HF_ARM_AES
;
400 if (probed
.hwcap2
& HWCAP2_PMULL
) hw
|= HF_ARM_PMULL
;
404 if (probed
.hwcap
& HWCAP_ASIMD
) hw
|= HF_ARM_NEON
;
405 if (probed
.hwcap
& HWCAP_AES
) hw
|= HF_ARM_AES
;
406 if (probed
.hwcap
& HWCAP_PMULL
) hw
|= HF_ARM_PMULL
;
409 /* Store the bitmask of features we probed for everyone to see. */
410 DISPATCH_STORE(hwcaps
, hw
);
412 /* Finally, make a report about the things we found. (Doing this earlier
413 * will pointlessly widen the window in which multiple threads will do the
414 * above auxiliary-vector probing.)
416 #define CAP__DEBUG(feat, tok) \
417 dispatch_debug("check auxv for feature `%s': %s", tok, \
418 hw & HF_##feat ? "available" : "absent");
423 /* --- @get_hwcaps@ --- *
427 * Returns: A mask of hardware capabilities and other features, as probed
428 * from the auxiliary vector.
431 static unsigned get_hwcaps(void)
435 DISPATCH_LOAD(hwcaps
, hw
);
436 if (!(hw
& HF_PROBED
)) { probe_hwcaps(); DISPATCH_LOAD(hwcaps
, hw
); }
442 /*----- External interface ------------------------------------------------*/
444 /* --- @dispatch_debug@ --- *
446 * Arguments: @const char *fmt@ = a format string
447 * @...@ = additional arguments
451 * Use: Writes a formatted message to standard output if dispatch
452 * debugging is enabled.
455 void dispatch_debug(const char *fmt
, ...)
458 const char *e
= getenv("CATACOMB_CPUDISPATCH_DEBUG");
460 if (e
&& *e
!= 'n' && *e
!= '0') {
462 fputs("Catacomb CPUDISPATCH: ", stderr
);
463 vfprintf(stderr
, fmt
, ap
);
469 /* --- @check_env@ --- *
471 * Arguments: @const char *ftok@ = feature token
473 * Returns: Zero if the feature is forced off; positive if it's forced
474 * on; negative if the user hasn't decided.
476 * Use: Checks the environment variable `CATACOMB_CPUFEAT' for the
477 * feature token @ftok@. The variable, if it exists, should be
478 * a space-separated sequence of `+tok' and `-tok' items. These
479 * tokens may end in `*', which matches any suffix.
482 static int IGNORABLE
check_env(const char *ftok
)
484 const char *p
, *q
, *pp
;
487 p
= getenv("CATACOMB_CPUFEAT");
491 while (ISSPACE(*p
)) p
++;
492 if (!*p
) return (-1);
494 case '+': d
= +1; p
++; break;
495 case '-': d
= 0; p
++; break;
496 default: d
= -1; break;
498 for (q
= p
; *q
&& !ISSPACE(*q
); q
++);
500 for (pp
= ftok
; p
< q
&& *pp
&& *p
== *pp
; p
++, pp
++);
501 if ((p
== q
&& !*pp
) || (*p
== '*' && p
+ 1 == q
)) return (d
);
508 /* --- @cpu_feature_p@ --- *
510 * Arguments: @unsigned feat@ = a @CPUFEAT_...@ code
512 * Returns: Nonzero if the feature is available.
518 feat_debug(const char *ftok
, const char *check
, int verdict
)
521 dispatch_debug("feature `%s': %s -> %s", ftok
, check
,
522 verdict ?
"available" : "absent");
527 int cpu_feature_p(int feat
)
531 #define CASE_CPUFEAT(feat, ftok, cond) case CPUFEAT_##feat: \
532 if ((f = feat_debug(ftok, "environment override", check_env(ftok))) >= 0) \
535 return (feat_debug(ftok, "runtime probe", cond));
538 #if CPUFAM_X86 || CPUFAM_AMD64
539 CASE_CPUFEAT(X86_SSE2
, "x86:sse2",
540 cpuid_feature_p(CPUID_1_D
, CPUID1D_SSE2
) &&
541 xmm_registers_available_p());
542 CASE_CPUFEAT(X86_AESNI
, "x86:aesni",
543 cpuid_feature_p(CPUID_1_C
, CPUID1C_AESNI
) &&
544 xmm_registers_available_p());
545 CASE_CPUFEAT(X86_RDRAND
, "x86:rdrand",
546 cpuid_feature_p(CPUID_1_C
, CPUID1C_RDRAND
) &&
547 rdrand_works_p(OP_RDRAND
));
548 CASE_CPUFEAT(X86_AVX
, "x86:avx",
549 cpuid_feature_p(CPUID_1_C
, CPUID1C_AVX
) &&
550 ymm_registers_available_p());
551 CASE_CPUFEAT(X86_AVX2
, "x86:avx2",
552 cpuid_feature_p(CPUID_1_C
, CPUID1C_AVX
) &&
553 cpuid_feature_p(CPUID_7_0_B
, CPUID70B_AVX2
) &&
554 ymm_registers_available_p());
555 CASE_CPUFEAT(X86_SSSE3
, "x86:ssse3",
556 cpuid_feature_p(CPUID_1_C
, CPUID1C_SSSE3
) &&
557 xmm_registers_available_p());
558 CASE_CPUFEAT(X86_PCLMUL
, "x86:pclmul",
559 cpuid_feature_p(CPUID_1_C
, CPUID1C_PCLMUL
) &&
560 xmm_registers_available_p());
561 CASE_CPUFEAT(X86_RDSEED
, "x86:rdseed",
562 cpuid_feature_p(CPUID_7_0_B
, CPUID70B_RDSEED
) &&
563 rdrand_works_p(OP_RDSEED
));
566 # define FEATP__CASE(feat, tok) \
567 CASE_CPUFEAT(feat, tok, get_hwcaps() & HF_##feat)
572 dispatch_debug("denying unknown feature %d", feat
);
578 /*----- That's all, folks -------------------------------------------------*/