3 * CPU-specific dispatch
5 * (c) 2015 Straylight/Edgeware
8 /*----- Licensing notice --------------------------------------------------*
10 * This file is part of Catacomb.
12 * Catacomb is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU Library General Public License as
14 * published by the Free Software Foundation; either version 2 of the
15 * License, or (at your option) any later version.
17 * Catacomb is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU Library General Public License for more details.
22 * You should have received a copy of the GNU Library General Public
23 * License along with Catacomb; if not, write to the Free
24 * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
28 /*----- Header files ------------------------------------------------------*/
38 #include <mLib/macros.h>
42 /*----- Intel x86/AMD64 feature probing -----------------------------------*/
44 #if CPUFAM_X86 || CPUFAM_AMD64
46 # define EFLAGS_ID (1u << 21)
47 # define CPUID1D_SSE2 (1u << 26)
48 # define CPUID1D_FXSR (1u << 24)
49 # define CPUID1C_AESNI (1u << 25)
51 struct cpuid
{ unsigned a
, b
, c
, d
; };
55 * Arguments: @struct cpuid *cc@ = where to write the result
56 * @unsigned a, c@ = EAX and ECX registers to set
60 * Use: Minimal C wrapper around the x86 `CPUID' instruction. Checks
61 * that the instruction is actually available before invoking
62 * it; fills the output structure with zero if it's not going to
68 static __inline__
unsigned getflags(void)
69 { unsigned f
; __asm__ ("pushf; popl %0" : "=g" (f
)); return (f
); }
70 static __inline__
unsigned setflags(unsigned f
)
73 __asm__ ("pushf; pushl %1; popf; pushf; popl %0; popf"
79 static __inline__
unsigned long getflags(void)
80 { unsigned long f
; __asm__ ("pushf; popq %0" : "=g" (f
)); return (f
); }
81 static __inline__
unsigned long long setflags(unsigned long f
)
84 __asm__ ("pushf; pushq %1; popf; pushf; popq %0; popf"
92 static void cpuid(struct cpuid
*cc
, unsigned a
, unsigned c
)
98 cc
->a
= cc
->b
= cc
->c
= cc
->d
= 0;
101 /* Stupid dance to detect whether the CPUID instruction is available. */
103 if (!(setflags(f
| EFLAGS_ID
) & EFLAGS_ID
) ||
104 setflags(f
& ~EFLAGS_ID
) & EFLAGS_ID
) {
105 dispatch_debug("CPUID instruction not available");
110 /* Alas, EBX is magical in PIC code, so abuse ESI instead. This isn't
111 * pretty, but it works.
114 __asm__ ("pushl %%ebx; cpuid; movl %%ebx, %%esi; popl %%ebx"
115 : "=a" (cc
->a
), "=S" (cc
->b
), "=c" (cc
->c
), "=d" (cc
->d
)
116 : "a" (a
) , "c" (c
));
118 __asm__ ("pushq %%rbx; cpuid; movl %%ebx, %%esi; popq %%rbx"
119 : "=a" (cc
->a
), "=S" (cc
->b
), "=c" (cc
->c
), "=d" (cc
->d
)
120 : "a" (a
) , "c" (c
));
122 # error "I'm confused."
124 dispatch_debug("CPUID(%08x, %08x) -> %08x, %08x, %08x, %08x",
125 a
, c
, cc
->a
, cc
->b
, cc
->c
, cc
->d
);
127 dispatch_debug("GNU inline assembler not available; can't CPUID");
131 static unsigned cpuid_maxleaf(void)
132 { struct cpuid c
; cpuid(&c
, 0, 0); return (c
.a
); }
134 /* --- @cpuid_features_p@ --- *
136 * Arguments: @unsigned dbits@ = bits to check in EDX
137 * @unsigned cbits@ = bits to check in ECX
139 * Returns: Nonzero if all the requested bits are set in the CPUID result
143 static int cpuid_features_p(unsigned dbits
, unsigned cbits
)
146 if (cpuid_maxleaf() < 1) return (0);
148 return ((c
.d
& dbits
) == dbits
&& (c
.c
& cbits
) == cbits
);
151 /* --- @xmm_registers_available_p@ --- *
155 * Returns: Nonzero if the operating system has made the XMM registers
159 static int xmm_registers_available_p(void)
163 /* This hack is by Agner Fog. Use FXSAVE/FXRSTOR to figure out whether the
164 * XMM registers are actually alive.
166 if (!cpuid_features_p(CPUID1D_FXSR
, 0)) return (0);
168 __asm__ ("movl %%esp, %%edx; subl $512, %%esp; andl $~15, %%esp\n"
170 "movl 160(%%esp), %%eax; xorl $0xaaaa5555, 160(%%esp)\n"
171 "fxrstor (%%esp); fxsave (%%esp)\n"
172 "movl 160(%%esp), %%ecx; movl %%eax, 160(%%esp)\n"
173 "fxrstor (%%esp); movl %%edx, %%esp\n"
179 __asm__ ("movq %%rsp, %%rdx; subq $512, %%rsp; andq $~15, %%rsp\n"
181 "movl 160(%%rsp), %%eax; xorl $0xaaaa5555, 160(%%rsp)\n"
182 "fxrstor (%%rsp); fxsave (%%rsp)\n"
183 "movl 160(%%rsp), %%ecx; movl %%eax, 160(%%rsp)\n"
184 "fxrstor (%%rsp); movq %%rdx, %%rsp\n"
190 # error "I'm confused."
192 dispatch_debug("XMM registers %savailable", f ?
"" : "not ");
195 dispatch_debug("GNU inline assembler not available; can't check for XMM");
202 /*----- General feature probing using auxiliary vectors -------------------*/
204 /* Try to find the system's definitions for auxiliary vector entries. */
205 #ifdef HAVE_SYS_AUXV_H
206 # include <sys/auxv.h>
208 # ifdef HAVE_LINUX_AUXVEC_H
209 # include <linux/auxvec.h>
211 # ifdef HAVE_ASM_HWCAP_H
212 # include <asm/hwcap.h>
216 /* The type of entries in the auxiliary vector. I'm assuming that `unsigned
217 * long' matches each platform's word length; if this is false then we'll
218 * need some host-specific tweaking here.
220 union auxval
{ long i
; unsigned long u
; const void *p
; };
221 struct auxentry
{ unsigned long type
; union auxval value
; };
223 /* Register each CPU family's interest in the auxiliary vector. Make sure
224 * that the necessary entry types are defined. This is primarily ordered by
225 * entry type to minimize duplication.
228 /* If we couldn't find any interesting entries then we can switch all of this
229 * machinery off. Also do that if we have no means for atomic updates.
231 #if WANT_ANY && CPU_DISPATCH_P
233 /* The main output of this section is a bitmask of detected features. The
234 * least significant bit will be set if we've tried to probe. Always access
235 * this using `DISPATCH_LOAD' and `DISPATCH_STORE'.
237 static unsigned hwcaps
= 0;
239 /* For each potentially interesting type which turned out not to exist or be
240 * wanted, define a dummy macro for the sake of the next step.
242 #ifndef WANT_AT_HWCAP
243 # define WANT_AT_HWCAP(_)
246 /* For each CPU family, define two lists.
248 * * `WANTAUX' is a list of the `WANT_AT_MUMBLE' macros which the CPU
249 * family tried to register interest in above. Each entry contains the
250 * interesting auxiliary vector entry type, the name of the union branch
251 * for its value, and the name of the slot in `struct auxprobe' in which
252 * to store the value.
254 * * `CAPMAP' is a list describing the output features which the CPU family
255 * intends to satisfy from the auxiliary vector. Each entry contains a
256 * feature name suffix, and the token name (for `check_env').
259 /* Build the bitmask for `hwcaps' from the `CAPMAP' list. */
262 #define HFI__ENUM(feat, tok) HFI_##feat,
269 #define HF__FLAG(feat, tok) HF_##feat = 1 << HFI_##feat,
275 /* Build a structure in which we can capture the interesting data from the
278 #define AUXUTYPE_i long
279 #define AUXUTYPE_u unsigned long
280 #define AUXUTYPE_p const void *
282 #define AUXPROBE__SLOT(type, ubranch, slot) AUXUTYPE_##ubranch slot;
283 WANTAUX(AUXPROBE__SLOT
)
287 /* --- @probe_hwcaps@ --- *
293 * Use: Attempt to find the auxiliary vector (which is well hidden)
294 * and discover interesting features from it.
297 static void probe_hwcaps(void)
299 unsigned hw
= HF_PROBED
;
300 struct auxprobe probed
= { 0 };
302 /* Populate `probed' with the information we manage to retrieve from the
303 * auxiliary vector. Slots we couldn't find are left zero-valued.
305 #if defined(HAVE_GETAUXVAL)
306 /* Shiny new libc lets us request individual entry types. This is almost
309 # define CAP__GET(type, slot, ubranch) \
310 probed.slot.ubranch = (AUXUTYPE_##ubranch)getauxval(type);
313 /* Otherwise we're a bit stuck, really. Modern Linux kernels make a copy
314 * of the vector available in `/procc' so we could try that.
316 * The usual place is stuck on the end of the environment vector, but that
317 * may well have moved, and we have no way of telling whether it has or
318 * whether there was ever an auxiliary vector there at all; so don't do
323 unsigned char *p
= 0, *q
= 0;
324 const struct auxentry
*a
;
327 /* Open the file and read it into a memory chunk. */
328 if ((fp
= fopen("/proc/self/auxv", "rb")) == 0) goto clean
;
330 if ((p
= malloc(sz
)) == 0) goto clean
;
332 n
= fread(p
+ off
, 1, sz
- off
, fp
);
335 sz
*= 2; if ((q
= realloc(p
, sz
)) == 0) break;
339 /* Work through the vector (or as much of it as we found) and extract the
340 * types we're interested in.
342 for (a
= (const struct auxentry
*)p
,
343 n
= sz
/sizeof(struct auxentry
);
346 #define CAP__SWITCH(type, ubranch, slot) \
347 case type: probed.slot = a->value.ubranch; break;
358 /* Each CPU family now has to pick through what was found and stashed in
359 * `probed', and set the appropriate flag bits in `hw'.
362 /* Store the bitmask of features we probed for everyone to see. */
363 DISPATCH_STORE(hwcaps
, hw
);
365 /* Finally, make a report about the things we found. (Doing this earlier
366 * will pointlessly widen the window in which multiple threads will do the
367 * above auxiliary-vector probing.)
369 #define CAP__DEBUG(feat, tok) \
370 dispatch_debug("check auxv for feature `%s': %s", tok, \
371 hw & HF_##feat ? "available" : "absent");
376 /* --- @get_hwcaps@ --- *
380 * Returns: A mask of hardware capabilities and other features, as probed
381 * from the auxiliary vector.
384 static unsigned get_hwcaps(void)
388 DISPATCH_LOAD(hwcaps
, hw
);
389 if (!(hwcaps
& HF_PROBED
)) { probe_hwcaps(); DISPATCH_LOAD(hwcaps
, hw
); }
395 /*----- External interface ------------------------------------------------*/
397 /* --- @dispatch_debug@ --- *
399 * Arguments: @const char *fmt@ = a format string
400 * @...@ = additional arguments
404 * Use: Writes a formatted message to standard output if dispatch
405 * debugging is enabled.
408 void dispatch_debug(const char *fmt
, ...)
411 const char *e
= getenv("CATACOMB_CPUDISPATCH_DEBUG");
413 if (e
&& *e
!= 'n' && *e
!= '0') {
415 fputs("Catacomb CPUDISPATCH: ", stderr
);
416 vfprintf(stderr
, fmt
, ap
);
422 /* --- @check_env@ --- *
424 * Arguments: @const char *ftok@ = feature token
426 * Returns: Zero if the feature is forced off; positive if it's forced
427 * on; negative if the user hasn't decided.
429 * Use: Checks the environment variable `CATACOMB_CPUFEAT' for the
430 * feature token @ftok@. The variable, if it exists, should be
431 * a space-separated sequence of `+tok' and `-tok' items. These
432 * tokens may end in `*', which matches any suffix.
435 static int IGNORABLE
check_env(const char *ftok
)
437 const char *p
, *q
, *pp
;
440 p
= getenv("CATACOMB_CPUFEAT");
444 while (isspace((unsigned char)*p
)) p
++;
445 if (!*p
) return (-1);
447 case '+': d
= +1; p
++; break;
448 case '-': d
= 0; p
++; break;
449 default: d
= -1; break;
451 for (q
= p
; *q
&& !isspace((unsigned char)*q
); q
++);
453 for (pp
= ftok
; p
< q
&& *pp
&& *p
== *pp
; p
++, pp
++);
454 if ((p
== q
&& !*pp
) || (*p
== '*' && p
+ 1 == q
)) return (d
);
461 /* --- @cpu_feature_p@ --- *
463 * Arguments: @unsigned feat@ = a @CPUFEAT_...@ code
465 * Returns: Nonzero if the feature is available.
471 feat_debug(const char *ftok
, const char *check
, int verdict
)
474 dispatch_debug("feature `%s': %s -> %s", ftok
, check
,
475 verdict ?
"available" : "absent");
480 int cpu_feature_p(int feat
)
484 #define CASE_CPUFEAT(feat, ftok, cond) case CPUFEAT_##feat: \
485 if ((f = feat_debug(ftok, "environment override", \
486 check_env(ftok))) >= 0) \
489 return (feat_debug(ftok, "runtime probe", cond));
492 #if CPUFAM_X86 || CPUFAM_AMD64
493 CASE_CPUFEAT(X86_SSE2
, "x86:sse2",
494 xmm_registers_available_p() &&
495 cpuid_features_p(CPUID1D_SSE2
, 0));
496 CASE_CPUFEAT(X86_AESNI
, "x86:aesni",
497 xmm_registers_available_p() &&
498 cpuid_features_p(CPUID1D_SSE2
, CPUID1C_AESNI
));
501 # define FEATP__CASE(feat, tok) \
502 CASE_CPUFEAT(feat, tok, get_hwcaps & HF_##feat)
507 dispatch_debug("denying unknown feature %d", feat
);
513 /*----- That's all, folks -------------------------------------------------*/