[catacomb] / base / dispatch.c

/* -*-c-*-
 *
 * CPU-specific dispatch
 *
 * (c) 2015 Straylight/Edgeware
 */

/*----- Licensing notice --------------------------------------------------*
 *
 * This file is part of Catacomb.
 *
 * Catacomb is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Library General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * Catacomb is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with Catacomb; if not, write to the Free
 * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
 * MA 02111-1307, USA.
 */

/*----- Header files ------------------------------------------------------*/

#include "config.h"

#include <ctype.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <mLib/macros.h>

#include "dispatch.h"

/*----- Intel x86/AMD64 feature probing -----------------------------------*/

#if CPUFAM_X86 || CPUFAM_AMD64

#  define EFLAGS_ID (1u << 21)
#  define CPUID1D_SSE2 (1u << 26)
#  define CPUID1D_FXSR (1u << 24)
#  define CPUID1C_AESNI (1u << 25)
#  define CPUID1C_RDRAND (1u << 30)

struct cpuid { unsigned a, b, c, d; };

/* --- @cpuid@ --- *
 *
 * Arguments:	@struct cpuid *cc@ = where to write the result
 *		@unsigned a, c@ = EAX and ECX registers to set
 *
 * Returns:	---
 *
 * Use:		Minimal C wrapper around the x86 `CPUID' instruction.  Checks
 *		that the instruction is actually available before invoking
 *		it; fills the output structure with zero if it's not going to
 *		work.
 */

#ifdef __GNUC__
#  if CPUFAM_X86
static __inline__ unsigned getflags(void)
  { unsigned f; __asm__ ("pushf; popl %0" : "=g" (f)); return (f); }
static __inline__ unsigned setflags(unsigned f)
{
  unsigned ff;
  __asm__ ("pushf; pushl %1; popf; pushf; popl %0; popf"
	   : "=r" (ff)
	   : "r" (f));
  return (ff);
}
#  else
static __inline__ unsigned long getflags(void)
  { unsigned long f; __asm__ ("pushf; popq %0" : "=g" (f)); return (f); }
static __inline__ unsigned long long setflags(unsigned long f)
{
  unsigned long ff;
  __asm__ ("pushf; pushq %1; popf; pushf; popq %0; popf"
	   : "=r" (ff)
	   : "r" (f));
  return (ff);
}
#  endif
#endif

static void cpuid(struct cpuid *cc, unsigned a, unsigned c)
{
#ifdef __GNUC__
  unsigned f;
#endif

  cc->a = cc->b = cc->c = cc->d = 0;

#ifdef __GNUC__
  /* Stupid dance to detect whether the CPUID instruction is available. */
  f = getflags();
  if (!(setflags(f |  EFLAGS_ID) & EFLAGS_ID) ||
	setflags(f & ~EFLAGS_ID) & EFLAGS_ID) {
    dispatch_debug("CPUID instruction not available");
    return;
  }
  setflags(f);

  /* Alas, EBX is magical in PIC code, so abuse ESI instead.  This isn't
   * pretty, but it works.
   */
#  if CPUFAM_X86
  __asm__ ("pushl %%ebx; cpuid; movl %%ebx, %%esi; popl %%ebx"
	   : "=a" (cc->a), "=S" (cc->b), "=c" (cc->c), "=d" (cc->d)
	   : "a" (a) , "c" (c));
#  elif CPUFAM_AMD64
  __asm__ ("pushq %%rbx; cpuid; movl %%ebx, %%esi; popq %%rbx"
	   : "=a" (cc->a), "=S" (cc->b), "=c" (cc->c), "=d" (cc->d)
	   : "a" (a) , "c" (c));
#  else
#    error "I'm confused."
#  endif
  dispatch_debug("CPUID(%08x, %08x) -> %08x, %08x, %08x, %08x",
		 a, c, cc->a, cc->b, cc->c, cc->d);
#else
  dispatch_debug("GNU inline assembler not available; can't CPUID");
#endif
}

static unsigned cpuid_maxleaf(void)
  { struct cpuid c; cpuid(&c, 0, 0); return (c.a); }

/* --- @cpuid_features_p@ --- *
 *
 * Arguments:	@unsigned dbits@ = bits to check in EDX
 *		@unsigned cbits@ = bits to check in ECX
 *
 * Returns:	Nonzero if all the requested bits are set in the CPUID result
 *		on leaf 1.
 */

static int cpuid_features_p(unsigned dbits, unsigned cbits)
{
  struct cpuid c;
  if (cpuid_maxleaf() < 1) return (0);
  cpuid(&c, 1, 0);
  return ((c.d & dbits) == dbits && (c.c & cbits) == cbits);
}

/* --- @xmm_registers_available_p@ --- *
 *
 * Arguments:	---
 *
 * Returns:	Nonzero if the operating system has made the XMM registers
 *		available for use.
 */

static int xmm_registers_available_p(void)
{
#ifdef __GNUC__
  unsigned f;
  /* This hack is by Agner Fog.  Use FXSAVE/FXRSTOR to figure out whether the
   * XMM registers are actually alive.
   */
  if (!cpuid_features_p(CPUID1D_FXSR, 0)) return (0);
#  if CPUFAM_X86
  __asm__ ("movl %%esp, %%edx; subl $512, %%esp; andl $~15, %%esp\n"
	   "fxsave (%%esp)\n"
	   "movl 160(%%esp), %%eax; xorl $0xaaaa5555, 160(%%esp)\n"
	   "fxrstor (%%esp); fxsave (%%esp)\n"
	   "movl 160(%%esp), %%ecx; movl %%eax, 160(%%esp)\n"
	   "fxrstor (%%esp); movl %%edx, %%esp\n"
	   "xorl %%ecx, %%eax"
	   : "=a" (f)
	   : /* no inputs */
	   : "%ecx", "%edx");
#  elif CPUFAM_AMD64
  __asm__ ("movq %%rsp, %%rdx; subq $512, %%rsp; andq $~15, %%rsp\n"
	   "fxsave (%%rsp)\n"
	   "movl 160(%%rsp), %%eax; xorl $0xaaaa5555, 160(%%rsp)\n"
	   "fxrstor (%%rsp); fxsave (%%rsp)\n"
	   "movl 160(%%rsp), %%ecx; movl %%eax, 160(%%rsp)\n"
	   "fxrstor (%%rsp); movq %%rdx, %%rsp\n"
	   "xorl %%ecx, %%eax"
	   : "=a" (f)
	   : /* no inputs */
	   : "%ecx", "%rdx");
#  else
#    error "I'm confused."
#  endif
  dispatch_debug("XMM registers %savailable", f ? "" : "not ");
  return (f);
#else
  dispatch_debug("GNU inline assembler not available; can't check for XMM");
  return (0);
#endif
}

/* --- @rdrand_works_p@ --- *
 *
 *
 * Arguments:	---
 *
 * Returns:	Nonzero if the `rdrand' instruction actually works.  Assumes
 *		that it's already been verified to be safe to issue.
 */

#ifdef __GNUC__
static int rdrand(unsigned *x)
{
  int i, rc;
  unsigned _t;

  i = 16;
  __asm__ ("" : "=g" (_t));
  __asm__ ("0: rdrand %2; jc 1f; decl %1; jnz 0b\n"
	   "mov $-1, %0; jmp 9f\n"
	   "1: movl %2, (%3); xorl %0, %0\n"
	   "9:"
	   : "=r" (rc), "+r" (i), "+r" (_t)
	   : "r" (x)
	   : "cc");
  return (rc);
}
#endif

static int rdrand_works_p(void)
{
  unsigned ref, x, i;

  /* Check that it doesn't always give the same answer.  Try four times: this
   * will fail with probability %$2^{-128}$% with a truly random generator,
   * which seems fair enough.
   */
  if (rdrand(&ref)) goto fail;
  for (i = 0; i < 4; i++) {
    if (rdrand(&x)) goto fail;
    if (x != ref) goto not_stuck;
  }
  dispatch_debug("RDRAND always returns 0x%08x!", ref);
  return (0);

not_stuck:
  dispatch_debug("RDRAND instruction looks plausible");
  return (1);

fail:
  dispatch_debug("RDRAND instruction fails too often");
  return (0);
}

#endif

/*----- General feature probing using auxiliary vectors -------------------*/

/* Try to find the system's definitions for auxiliary vector entries. */
#ifdef HAVE_SYS_AUXV_H
#  include <sys/auxv.h>
#endif
#ifdef HAVE_LINUX_AUXVEC_H
#  include <linux/auxvec.h>
#endif
#ifdef HAVE_ASM_HWCAP_H
#  include <asm/hwcap.h>
#endif

/* The type of entries in the auxiliary vector.  I'm assuming that `unsigned
 * long' matches each platform's word length; if this is false then we'll
 * need some host-specific tweaking here.
 */
union auxval { long i; unsigned long u; const void *p; };
struct auxentry { unsigned long type; union auxval value; };

/* Register each CPU family's interest in the auxiliary vector.  Make sure
 * that the necessary entry types are defined.  This is primarily ordered by
 * entry type to minimize duplication.
 */
#if defined(AT_HWCAP) && CPUFAM_ARMEL
#  define WANT_ANY 1
#  define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
#endif

#if defined(AT_HWCAP) && CPUFAM_ARM64
#  define WANT_ANY 1
#  define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
#endif

#if defined(AT_HWCAP2) && CPUFAM_ARMEL
#  define WANT_ANY 1
#  define WANT_AT_HWCAP2(_) _(AT_HWCAP2, u, hwcap2)
#endif

/* If we couldn't find any interesting entries then we can switch all of this
 * machinery off.  Also do that if we have no means for atomic updates.
 */
#if WANT_ANY && CPU_DISPATCH_P

/* The main output of this section is a bitmask of detected features.  The
 * least significant bit will be set if we've tried to probe.  Always access
 * this using `DISPATCH_LOAD' and `DISPATCH_STORE'.
 */
static unsigned hwcaps = 0;

/* For each potentially interesting type which turned out not to exist or be
 * wanted, define a dummy macro for the sake of the next step.
 */
#ifndef WANT_AT_HWCAP
#  define WANT_AT_HWCAP(_)
#endif
#ifndef WANT_AT_HWCAP2
#  define WANT_AT_HWCAP2(_)
#endif

/* For each CPU family, define two lists.
 *
 *   * `WANTAUX' is a list of the `WANT_AT_MUMBLE' macros which the CPU
 *     family tried to register interest in above.  Each entry contains the
 *     interesting auxiliary vector entry type, the name of the union branch
 *     for its value, and the name of the slot in `struct auxprobe' in which
 *     to store the value.
 *
 *   * `CAPMAP' is a list describing the output features which the CPU family
 *     intends to satisfy from the auxiliary vector.  Each entry contains a
 *     feature name suffix, and the token name (for `check_env').
 */
#if CPUFAM_ARMEL
#  define WANTAUX(_)							\
	WANT_AT_HWCAP(_)						\
	WANT_AT_HWCAP2(_)
#  define CAPMAP(_)							\
	_(ARM_VFP, "arm:vfp")						\
	_(ARM_NEON, "arm:neon")						\
	_(ARM_V4, "arm:v4")						\
	_(ARM_D32, "arm:d32")						\
	_(ARM_AES, "arm:aes")
#endif
#if CPUFAM_ARM64
#  define WANTAUX(_)							\
	WANT_AT_HWCAP(_)
#  define CAPMAP(_)							\
	_(ARM_AES, "arm:aes")
#endif

/* Build the bitmask for `hwcaps' from the `CAPMAP' list. */
enum {
  HFI_PROBED = 0,
#define HFI__ENUM(feat, tok) HFI_##feat,
  CAPMAP(HFI__ENUM)
#undef HFI__ENUM
  HFI__END
};
enum {
  HF_PROBED = 1,
#define HF__FLAG(feat, tok) HF_##feat = 1 << HFI_##feat,
  CAPMAP(HF__FLAG)
#undef HF__FLAG
  HF__END
};

/* Build a structure in which we can capture the interesting data from the
 * auxiliary vector.
 */
#define AUXUTYPE_i long
#define AUXUTYPE_u unsigned long
#define AUXUTYPE_p const void *
struct auxprobe {
#define AUXPROBE__SLOT(type, ubranch, slot) AUXUTYPE_##ubranch slot;
  WANTAUX(AUXPROBE__SLOT)
#undef AUXPROBE_SLOT
};

/* --- @probe_hwcaps@ --- *
 *
 * Arguments:	---
 *
 * Returns:	---
 *
 * Use:		Attempt to find the auxiliary vector (which is well hidden)
 *		and discover interesting features from it.
 */

static void probe_hwcaps(void)
{
  unsigned hw = HF_PROBED;
  struct auxprobe probed = { 0 };

  /* Populate `probed' with the information we manage to retrieve from the
   * auxiliary vector.  Slots we couldn't find are left zero-valued.
   */
#if defined(HAVE_GETAUXVAL)
  /* Shiny new libc lets us request individual entry types.  This is almost
   * too easy.
   */
#  define CAP__GET(type, ubranch, slot)					\
	probed.slot = (AUXUTYPE_##ubranch)getauxval(type);
  WANTAUX(CAP__GET)
#else
  /* Otherwise we're a bit stuck, really.  Modern Linux kernels make a copy
   * of the vector available in `/procc' so we could try that.
   *
   * The usual place is stuck on the end of the environment vector, but that
   * may well have moved, and we have no way of telling whether it has or
   * whether there was ever an auxiliary vector there at all; so don't do
   * that.
   */
  {
    FILE *fp = 0;
    unsigned char *p = 0, *q = 0;
    const struct auxentry *a;
    size_t sz, off, n;

    /* Open the file and read it into a memory chunk. */
    if ((fp = fopen("/proc/self/auxv", "rb")) == 0) goto clean;
    sz = 4096; off = 0;
    if ((p = malloc(sz)) == 0) goto clean;
    for (;;) {
      n = fread(p + off, 1, sz - off, fp);
      off += n;
      if (off < sz) break;
      sz *= 2; if ((q = realloc(p, sz)) == 0) break;
      p = q;
    }

    /* Work through the vector (or as much of it as we found) and extract the
     * types we're interested in.
     */
    for (a = (const struct auxentry *)p,
	   n = sz/sizeof(struct auxentry);
	 n--; a++) {
      switch (a->type) {
#define CAP__SWITCH(type, ubranch, slot)				\
	case type: probed.slot = a->value.ubranch; break;
	WANTAUX(CAP__SWITCH)
	case AT_NULL: goto clean;
      }
    }

  clean:
    if (p) free(p);
    if (fp) fclose(fp);
  }
#endif

  /* Each CPU family now has to pick through what was found and stashed in
   * `probed', and set the appropriate flag bits in `hw'.
   */
#if CPUFAM_ARMEL
  if (probed.hwcap & HWCAP_VFPv3) hw |= HF_ARM_VFP;
  if (probed.hwcap & HWCAP_NEON) hw |= HF_ARM_NEON;
  if (probed.hwcap & HWCAP_VFPD32) hw |= HF_ARM_D32;
  if (probed.hwcap & HWCAP_VFPv4) hw |= HF_ARM_V4;
#  ifdef HWCAP2_AES
  if (probed.hwcap2 & HWCAP2_AES) hw |= HF_ARM_AES;
#  endif
#endif
#if CPUFAM_ARM64
  if (probed.hwcap & HWCAP_AES) hw |= HF_ARM_AES;
#endif

  /* Store the bitmask of features we probed for everyone to see. */
  DISPATCH_STORE(hwcaps, hw);

  /* Finally, make a report about the things we found.  (Doing this earlier
   * will pointlessly widen the window in which multiple threads will do the
   * above auxiliary-vector probing.)
   */
#define CAP__DEBUG(feat, tok)						\
  dispatch_debug("check auxv for feature `%s': %s", tok,		\
		 hw & HF_##feat ? "available" : "absent");
  CAPMAP(CAP__DEBUG)
#undef CAP__DEBUG
}

/* --- @get_hwcaps@ --- *
 *
 * Arguments:	---
 *
 * Returns:	A mask of hardware capabilities and other features, as probed
 *		from the auxiliary vector.
 */

static unsigned get_hwcaps(void)
{
  unsigned hw;

  DISPATCH_LOAD(hwcaps, hw);
  if (!(hwcaps & HF_PROBED)) { probe_hwcaps(); DISPATCH_LOAD(hwcaps, hw); }
  return (hw);
}

#endif

/*----- External interface ------------------------------------------------*/

/* --- @dispatch_debug@ --- *
 *
 * Arguments:	@const char *fmt@ = a format string
 *		@...@ = additional arguments
 *
 * Returns:	---
 *
 * Use:		Writes a formatted message to standard output if dispatch
 *		debugging is enabled.
 */

void dispatch_debug(const char *fmt, ...)
{
  va_list ap;
  const char *e = getenv("CATACOMB_CPUDISPATCH_DEBUG");

  if (e && *e != 'n' && *e != '0') {
    va_start(ap, fmt);
    fputs("Catacomb CPUDISPATCH: ", stderr);
    vfprintf(stderr, fmt, ap);
    fputc('\n', stderr);
    va_end(ap);
  }
}

/* --- @check_env@ --- *
 *
 * Arguments:	@const char *ftok@ = feature token
 *
 * Returns:	Zero if the feature is forced off; positive if it's forced
 *		on; negative if the user hasn't decided.
 *
 * Use:		Checks the environment variable `CATACOMB_CPUFEAT' for the
 *		feature token @ftok@.  The variable, if it exists, should be
 *		a space-separated sequence of `+tok' and `-tok' items.  These
 *		tokens may end in `*', which matches any suffix.
 */

static int IGNORABLE check_env(const char *ftok)
{
  const char *p, *q, *pp;
  int d;

  p = getenv("CATACOMB_CPUFEAT");
  if (!p) return (-1);

  for (;;) {
    while (isspace((unsigned char)*p)) p++;
    if (!*p) return (-1);
    switch (*p) {
      case '+': d = +1; p++; break;
      case '-': d =  0; p++; break;
      default:  d = -1;      break;
    }
    for (q = p; *q && !isspace((unsigned char)*q); q++);
    if (d >= 0) {
      for (pp = ftok; p < q && *pp && *p == *pp; p++, pp++);
      if ((p == q && !*pp) || (*p == '*' && p + 1 == q)) return (d);
    }
    p = q;
  }
  return (-1);
}

/* --- @cpu_feature_p@ --- *
 *
 * Arguments:	@unsigned feat@ = a @CPUFEAT_...@ code
 *
 * Returns:	Nonzero if the feature is available.
 */

#include <stdio.h>

static int IGNORABLE
  feat_debug(const char *ftok, const char *check, int verdict)
{
  if (verdict >= 0) {
    dispatch_debug("feature `%s': %s -> %s", ftok, check,
		   verdict ? "available" : "absent");
  }
  return (verdict);
}

int cpu_feature_p(int feat)
{
  int IGNORABLE f;
  IGNORE(f);
#define CASE_CPUFEAT(feat, ftok, cond) case CPUFEAT_##feat:		\
  if ((f = feat_debug(ftok, "environment override",			\
		      check_env(ftok))) >= 0)				\
    return (f);								\
  else									\
    return (feat_debug(ftok, "runtime probe", cond));

  switch (feat) {
#if CPUFAM_X86 || CPUFAM_AMD64
    CASE_CPUFEAT(X86_SSE2, "x86:sse2",
		 cpuid_features_p(CPUID1D_SSE2, 0) &&
		 xmm_registers_available_p());
    CASE_CPUFEAT(X86_AESNI, "x86:aesni",
		 cpuid_features_p(CPUID1D_SSE2, CPUID1C_AESNI) &&
		 xmm_registers_available_p());
    CASE_CPUFEAT(X86_RDRAND, "x86:rdrand",
		 cpuid_features_p(0, CPUID1C_RDRAND) && rdrand_works_p());
#endif
#ifdef CAPMAP
#  define FEATP__CASE(feat, tok)					\
	CASE_CPUFEAT(feat, tok,	get_hwcaps() & HF_##feat)
    CAPMAP(FEATP__CASE)
#undef FEATP__CASE
#endif
    default:
      dispatch_debug("denying unknown feature %d", feat);
      return (0);
  }
#undef CASE_CPUFEAT
}

/*----- That's all, folks -------------------------------------------------*/
Commit	Line	Data
08e2be29 MW	1	/* --c--
	2	*
	3	* CPU-specific dispatch
	4	*
	5	* (c) 2015 Straylight/Edgeware
	6	*/
	7
	8	/----- Licensing notice --------------------------------------------------
	9	*
	10	* This file is part of Catacomb.
	11	*
	12	* Catacomb is free software; you can redistribute it and/or modify
	13	* it under the terms of the GNU Library General Public License as
	14	* published by the Free Software Foundation; either version 2 of the
	15	* License, or (at your option) any later version.
	16	*
	17	* Catacomb is distributed in the hope that it will be useful,
	18	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	20	* GNU Library General Public License for more details.
	21	*
	22	* You should have received a copy of the GNU Library General Public
	23	* License along with Catacomb; if not, write to the Free
	24	* Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
	25	* MA 02111-1307, USA.
	26	*/
	27
	28	/----- Header files ------------------------------------------------------/
	29
	30	#include "config.h"
	31
	32	#include <ctype.h>
fac645f7 MW	33	#include <stdarg.h>
fac645f7 MW	34	#include <stdio.h>
08e2be29 MW	35	#include <stdlib.h>
	36	#include <string.h>
	37
	38	#include <mLib/macros.h>
	39
	40	#include "dispatch.h"
	41
d26ad211	42	/----- Intel x86/AMD64 feature probing -----------------------------------/
08e2be29	43
0f23f75f	44	#if CPUFAM_X86 \|\| CPUFAM_AMD64
08e2be29	45
7680f863 MW	46	# define EFLAGS_ID (1u << 21)
	47	# define CPUID1D_SSE2 (1u << 26)
	48	# define CPUID1D_FXSR (1u << 24)
	49	# define CPUID1C_AESNI (1u << 25)
d25653be	50	# define CPUID1C_RDRAND (1u << 30)
08e2be29 MW	51
	52	struct cpuid { unsigned a, b, c, d; };
	53
	54	/* --- @cpuid@ --- *
	55	*
	56	* Arguments: @struct cpuid *cc@ = where to write the result
	57	* @unsigned a, c@ = EAX and ECX registers to set
	58	*
	59	* Returns: ---
	60	*
	61	* Use: Minimal C wrapper around the x86 `CPUID' instruction. Checks
	62	* that the instruction is actually available before invoking
	63	* it; fills the output structure with zero if it's not going to
	64	* work.
	65	*/
	66
	67	#ifdef __GNUC__
0f23f75f	68	# if CPUFAM_X86
08e2be29 MW	69	static __inline__ unsigned getflags(void)
	70	{ unsigned f; __asm__ ("pushf; popl %0" : "=g" (f)); return (f); }
	71	static __inline__ unsigned setflags(unsigned f)
	72	{
	73	unsigned ff;
	74	__asm__ ("pushf; pushl %1; popf; pushf; popl %0; popf"
25eccdf7 MW	75	: "=r" (ff)
25eccdf7 MW	76	: "r" (f));
08e2be29 MW	77	return (ff);
08e2be29 MW	78	}
0f23f75f MW	79	# else
	80	static __inline__ unsigned long getflags(void)
	81	{ unsigned long f; __asm__ ("pushf; popq %0" : "=g" (f)); return (f); }
	82	static __inline__ unsigned long long setflags(unsigned long f)
	83	{
	84	unsigned long ff;
	85	__asm__ ("pushf; pushq %1; popf; pushf; popq %0; popf"
25eccdf7 MW	86	: "=r" (ff)
25eccdf7 MW	87	: "r" (f));
0f23f75f MW	88	return (ff);
	89	}
	90	# endif
08e2be29 MW	91	#endif
	92
	93	static void cpuid(struct cpuid *cc, unsigned a, unsigned c)
	94	{
	95	#ifdef __GNUC__
	96	unsigned f;
	97	#endif
	98
	99	cc->a = cc->b = cc->c = cc->d = 0;
	100
	101	#ifdef __GNUC__
	102	/* Stupid dance to detect whether the CPUID instruction is available. */
	103	f = getflags();
fac645f7 MW	104	if (!(setflags(f \| EFLAGS_ID) & EFLAGS_ID) \|\|
	105	setflags(f & ~EFLAGS_ID) & EFLAGS_ID) {
	106	dispatch_debug("CPUID instruction not available");
	107	return;
	108	}
08e2be29 MW	109	setflags(f);
	110
	111	/* Alas, EBX is magical in PIC code, so abuse ESI instead. This isn't
	112	* pretty, but it works.
	113	*/
0f23f75f	114	# if CPUFAM_X86
08e2be29 MW	115	__asm__ ("pushl %%ebx; cpuid; movl %%ebx, %%esi; popl %%ebx"
	116	: "=a" (cc->a), "=S" (cc->b), "=c" (cc->c), "=d" (cc->d)
	117	: "a" (a) , "c" (c));
0f23f75f MW	118	# elif CPUFAM_AMD64
	119	__asm__ ("pushq %%rbx; cpuid; movl %%ebx, %%esi; popq %%rbx"
	120	: "=a" (cc->a), "=S" (cc->b), "=c" (cc->c), "=d" (cc->d)
	121	: "a" (a) , "c" (c));
	122	# else
	123	# error "I'm confused."
	124	# endif
	125	dispatch_debug("CPUID(%08x, %08x) -> %08x, %08x, %08x, %08x",
	126	a, c, cc->a, cc->b, cc->c, cc->d);
fac645f7 MW	127	#else
fac645f7 MW	128	dispatch_debug("GNU inline assembler not available; can't CPUID");
08e2be29 MW	129	#endif
	130	}
	131
	132	static unsigned cpuid_maxleaf(void)
	133	{ struct cpuid c; cpuid(&c, 0, 0); return (c.a); }
	134
11b17977 MW	135	/* --- @cpuid_features_p@ --- *
	136	*
	137	* Arguments: @unsigned dbits@ = bits to check in EDX
	138	* @unsigned cbits@ = bits to check in ECX
	139	*
	140	* Returns: Nonzero if all the requested bits are set in the CPUID result
	141	* on leaf 1.
	142	*/
	143
08e2be29 MW	144	static int cpuid_features_p(unsigned dbits, unsigned cbits)
	145	{
	146	struct cpuid c;
	147	if (cpuid_maxleaf() < 1) return (0);
	148	cpuid(&c, 1, 0);
	149	return ((c.d & dbits) == dbits && (c.c & cbits) == cbits);
	150	}
	151
11b17977 MW	152	/* --- @xmm_registers_available_p@ --- *
	153	*
	154	* Arguments: ---
	155	*
	156	* Returns: Nonzero if the operating system has made the XMM registers
	157	* available for use.
	158	*/
	159
acbe16df MW	160	static int xmm_registers_available_p(void)
	161	{
	162	#ifdef __GNUC__
	163	unsigned f;
	164	/* This hack is by Agner Fog. Use FXSAVE/FXRSTOR to figure out whether the
	165	* XMM registers are actually alive.
	166	*/
	167	if (!cpuid_features_p(CPUID1D_FXSR, 0)) return (0);
0f23f75f	168	# if CPUFAM_X86
acbe16df MW	169	__asm__ ("movl %%esp, %%edx; subl $512, %%esp; andl $~15, %%esp\n"
	170	"fxsave (%%esp)\n"
	171	"movl 160(%%esp), %%eax; xorl $0xaaaa5555, 160(%%esp)\n"
	172	"fxrstor (%%esp); fxsave (%%esp)\n"
	173	"movl 160(%%esp), %%ecx; movl %%eax, 160(%%esp)\n"
	174	"fxrstor (%%esp); movl %%edx, %%esp\n"
	175	"xorl %%ecx, %%eax"
	176	: "=a" (f)
	177	: /* no inputs */
	178	: "%ecx", "%edx");
0f23f75f MW	179	# elif CPUFAM_AMD64
	180	__asm__ ("movq %%rsp, %%rdx; subq $512, %%rsp; andq $~15, %%rsp\n"
	181	"fxsave (%%rsp)\n"
	182	"movl 160(%%rsp), %%eax; xorl $0xaaaa5555, 160(%%rsp)\n"
	183	"fxrstor (%%rsp); fxsave (%%rsp)\n"
	184	"movl 160(%%rsp), %%ecx; movl %%eax, 160(%%rsp)\n"
	185	"fxrstor (%%rsp); movq %%rdx, %%rsp\n"
	186	"xorl %%ecx, %%eax"
	187	: "=a" (f)
	188	: /* no inputs */
	189	: "%ecx", "%rdx");
	190	# else
	191	# error "I'm confused."
	192	# endif
	193	dispatch_debug("XMM registers %savailable", f ? "" : "not ");
acbe16df MW	194	return (f);
acbe16df MW	195	#else
fac645f7	196	dispatch_debug("GNU inline assembler not available; can't check for XMM");
acbe16df MW	197	return (0);
	198	#endif
	199	}
	200
35b1eba8 MW	201	/* --- @rdrand_works_p@ --- *
	202	*
	203	*
	204	* Arguments: ---
	205	*
	206	* Returns: Nonzero if the `rdrand' instruction actually works. Assumes
	207	* that it's already been verified to be safe to issue.
	208	*/
	209
	210	#ifdef __GNUC__
	211	static int rdrand(unsigned *x)
	212	{
	213	int i, rc;
	214	unsigned _t;
	215
	216	i = 16;
	217	__asm__ ("" : "=g" (_t));
	218	__asm__ ("0: rdrand %2; jc 1f; decl %1; jnz 0b\n"
	219	"mov $-1, %0; jmp 9f\n"
	220	"1: movl %2, (%3); xorl %0, %0\n"
	221	"9:"
	222	: "=r" (rc), "+r" (i), "+r" (_t)
	223	: "r" (x)
	224	: "cc");
	225	return (rc);
	226	}
	227	#endif
	228
	229	static int rdrand_works_p(void)
	230	{
	231	unsigned ref, x, i;
	232
	233	/* Check that it doesn't always give the same answer. Try four times: this
	234	* will fail with probability %$2^{-128}$% with a truly random generator,
	235	* which seems fair enough.
	236	*/
	237	if (rdrand(&ref)) goto fail;
	238	for (i = 0; i < 4; i++) {
	239	if (rdrand(&x)) goto fail;
	240	if (x != ref) goto not_stuck;
	241	}
	242	dispatch_debug("RDRAND always returns 0x%08x!", ref);
	243	return (0);
	244
	245	not_stuck:
	246	dispatch_debug("RDRAND instruction looks plausible");
	247	return (1);
	248
	249	fail:
	250	dispatch_debug("RDRAND instruction fails too often");
	251	return (0);
	252	}
	253
08e2be29 MW	254	#endif
08e2be29 MW	255
a02a22d4 MW	256	/----- General feature probing using auxiliary vectors -------------------/
	257
	258	/* Try to find the system's definitions for auxiliary vector entries. */
	259	#ifdef HAVE_SYS_AUXV_H
	260	# include <sys/auxv.h>
a0e9bb8a MW	261	#endif
	262	#ifdef HAVE_LINUX_AUXVEC_H
	263	# include <linux/auxvec.h>
	264	#endif
	265	#ifdef HAVE_ASM_HWCAP_H
	266	# include <asm/hwcap.h>
a02a22d4 MW	267	#endif
	268
	269	/* The type of entries in the auxiliary vector. I'm assuming that `unsigned
	270	* long' matches each platform's word length; if this is false then we'll
	271	* need some host-specific tweaking here.
	272	*/
	273	union auxval { long i; unsigned long u; const void *p; };
	274	struct auxentry { unsigned long type; union auxval value; };
	275
	276	/* Register each CPU family's interest in the auxiliary vector. Make sure
	277	* that the necessary entry types are defined. This is primarily ordered by
	278	* entry type to minimize duplication.
	279	*/
61bd904b MW	280	#if defined(AT_HWCAP) && CPUFAM_ARMEL
	281	# define WANT_ANY 1
	282	# define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
	283	#endif
a02a22d4	284
e492db88 MW	285	#if defined(AT_HWCAP) && CPUFAM_ARM64
	286	# define WANT_ANY 1
	287	# define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
	288	#endif
	289
26e182fc MW	290	#if defined(AT_HWCAP2) && CPUFAM_ARMEL
	291	# define WANT_ANY 1
	292	# define WANT_AT_HWCAP2(_) _(AT_HWCAP2, u, hwcap2)
	293	#endif
	294
a02a22d4 MW	295	/* If we couldn't find any interesting entries then we can switch all of this
	296	* machinery off. Also do that if we have no means for atomic updates.
	297	*/
	298	#if WANT_ANY && CPU_DISPATCH_P
	299
	300	/* The main output of this section is a bitmask of detected features. The
	301	* least significant bit will be set if we've tried to probe. Always access
	302	* this using `DISPATCH_LOAD' and `DISPATCH_STORE'.
	303	*/
	304	static unsigned hwcaps = 0;
	305
	306	/* For each potentially interesting type which turned out not to exist or be
	307	* wanted, define a dummy macro for the sake of the next step.
	308	*/
	309	#ifndef WANT_AT_HWCAP
	310	# define WANT_AT_HWCAP(_)
	311	#endif
f013a4f2 MW	312	#ifndef WANT_AT_HWCAP2
	313	# define WANT_AT_HWCAP2(_)
	314	#endif
a02a22d4 MW	315
	316	/* For each CPU family, define two lists.
	317	*
	318	* * `WANTAUX' is a list of the `WANT_AT_MUMBLE' macros which the CPU
	319	* family tried to register interest in above. Each entry contains the
	320	* interesting auxiliary vector entry type, the name of the union branch
	321	* for its value, and the name of the slot in `struct auxprobe' in which
	322	* to store the value.
	323	*
	324	* * `CAPMAP' is a list describing the output features which the CPU family
	325	* intends to satisfy from the auxiliary vector. Each entry contains a
	326	* feature name suffix, and the token name (for `check_env').
	327	*/
61bd904b MW	328	#if CPUFAM_ARMEL
61bd904b MW	329	# define WANTAUX(_) \
26e182fc MW	330	WANT_AT_HWCAP(_) \
26e182fc MW	331	WANT_AT_HWCAP2(_)
61bd904b MW	332	# define CAPMAP(_) \
	333	_(ARM_VFP, "arm:vfp") \
	334	_(ARM_NEON, "arm:neon") \
	335	_(ARM_V4, "arm:v4") \
26e182fc MW	336	_(ARM_D32, "arm:d32") \
26e182fc MW	337	_(ARM_AES, "arm:aes")
61bd904b	338	#endif
e492db88 MW	339	#if CPUFAM_ARM64
	340	# define WANTAUX(_) \
	341	WANT_AT_HWCAP(_)
	342	# define CAPMAP(_) \
	343	_(ARM_AES, "arm:aes")
	344	#endif
a02a22d4 MW	345
	346	/* Build the bitmask for `hwcaps' from the `CAPMAP' list. */
	347	enum {
	348	HFI_PROBED = 0,
	349	#define HFI__ENUM(feat, tok) HFI_##feat,
	350	CAPMAP(HFI__ENUM)
	351	#undef HFI__ENUM
	352	HFI__END
	353	};
	354	enum {
	355	HF_PROBED = 1,
	356	#define HF__FLAG(feat, tok) HF_##feat = 1 << HFI_##feat,
	357	CAPMAP(HF__FLAG)
	358	#undef HF__FLAG
	359	HF__END
	360	};
	361
	362	/* Build a structure in which we can capture the interesting data from the
	363	* auxiliary vector.
	364	*/
	365	#define AUXUTYPE_i long
	366	#define AUXUTYPE_u unsigned long
	367	#define AUXUTYPE_p const void *
	368	struct auxprobe {
	369	#define AUXPROBE__SLOT(type, ubranch, slot) AUXUTYPE_##ubranch slot;
	370	WANTAUX(AUXPROBE__SLOT)
	371	#undef AUXPROBE_SLOT
	372	};
	373
	374	/* --- @probe_hwcaps@ --- *
	375	*
	376	* Arguments: ---
	377	*
	378	* Returns: ---
	379	*
	380	* Use: Attempt to find the auxiliary vector (which is well hidden)
	381	* and discover interesting features from it.
	382	*/
	383
	384	static void probe_hwcaps(void)
	385	{
	386	unsigned hw = HF_PROBED;
	387	struct auxprobe probed = { 0 };
	388
	389	/* Populate `probed' with the information we manage to retrieve from the
	390	* auxiliary vector. Slots we couldn't find are left zero-valued.
	391	*/
	392	#if defined(HAVE_GETAUXVAL)
	393	/* Shiny new libc lets us request individual entry types. This is almost
	394	* too easy.
	395	*/
84850eec MW	396	# define CAP__GET(type, ubranch, slot) \
84850eec MW	397	probed.slot = (AUXUTYPE_##ubranch)getauxval(type);
a02a22d4 MW	398	WANTAUX(CAP__GET)
	399	#else
	400	/* Otherwise we're a bit stuck, really. Modern Linux kernels make a copy
	401	* of the vector available in `/procc' so we could try that.
	402	*
	403	* The usual place is stuck on the end of the environment vector, but that
	404	* may well have moved, and we have no way of telling whether it has or
	405	* whether there was ever an auxiliary vector there at all; so don't do
	406	* that.
	407	*/
	408	{
	409	FILE *fp = 0;
	410	unsigned char p = 0, q = 0;
	411	const struct auxentry *a;
	412	size_t sz, off, n;
	413
	414	/* Open the file and read it into a memory chunk. */
	415	if ((fp = fopen("/proc/self/auxv", "rb")) == 0) goto clean;
	416	sz = 4096; off = 0;
	417	if ((p = malloc(sz)) == 0) goto clean;
	418	for (;;) {
	419	n = fread(p + off, 1, sz - off, fp);
	420	off += n;
	421	if (off < sz) break;
	422	sz *= 2; if ((q = realloc(p, sz)) == 0) break;
	423	p = q;
	424	}
	425
	426	/* Work through the vector (or as much of it as we found) and extract the
	427	* types we're interested in.
	428	*/
	429	for (a = (const struct auxentry *)p,
	430	n = sz/sizeof(struct auxentry);
	431	n--; a++) {
	432	switch (a->type) {
	433	#define CAP__SWITCH(type, ubranch, slot) \
	434	case type: probed.slot = a->value.ubranch; break;
	435	WANTAUX(CAP__SWITCH)
dfcb2a0b	436	case AT_NULL: goto clean;
a02a22d4 MW	437	}
	438	}
	439
	440	clean:
	441	if (p) free(p);
	442	if (fp) fclose(fp);
	443	}
	444	#endif
	445
	446	/* Each CPU family now has to pick through what was found and stashed in
	447	* `probed', and set the appropriate flag bits in `hw'.
	448	*/
61bd904b MW	449	#if CPUFAM_ARMEL
	450	if (probed.hwcap & HWCAP_VFPv3) hw \|= HF_ARM_VFP;
	451	if (probed.hwcap & HWCAP_NEON) hw \|= HF_ARM_NEON;
	452	if (probed.hwcap & HWCAP_VFPD32) hw \|= HF_ARM_D32;
	453	if (probed.hwcap & HWCAP_VFPv4) hw \|= HF_ARM_V4;
26e182fc MW	454	# ifdef HWCAP2_AES
	455	if (probed.hwcap2 & HWCAP2_AES) hw \|= HF_ARM_AES;
	456	# endif
61bd904b	457	#endif
e492db88 MW	458	#if CPUFAM_ARM64
	459	if (probed.hwcap & HWCAP_AES) hw \|= HF_ARM_AES;
	460	#endif
a02a22d4 MW	461
	462	/* Store the bitmask of features we probed for everyone to see. */
	463	DISPATCH_STORE(hwcaps, hw);
	464
	465	/* Finally, make a report about the things we found. (Doing this earlier
	466	* will pointlessly widen the window in which multiple threads will do the
	467	* above auxiliary-vector probing.)
	468	*/
	469	#define CAP__DEBUG(feat, tok) \
	470	dispatch_debug("check auxv for feature `%s': %s", tok, \
	471	hw & HF_##feat ? "available" : "absent");
	472	CAPMAP(CAP__DEBUG)
	473	#undef CAP__DEBUG
	474	}
	475
	476	/* --- @get_hwcaps@ --- *
	477	*
	478	* Arguments: ---
	479	*
	480	* Returns: A mask of hardware capabilities and other features, as probed
	481	* from the auxiliary vector.
	482	*/
	483
	484	static unsigned get_hwcaps(void)
	485	{
	486	unsigned hw;
	487
	488	DISPATCH_LOAD(hwcaps, hw);
	489	if (!(hwcaps & HF_PROBED)) { probe_hwcaps(); DISPATCH_LOAD(hwcaps, hw); }
	490	return (hw);
	491	}
	492
	493	#endif
	494
d26ad211 MW	495	/----- External interface ------------------------------------------------/
d26ad211 MW	496
fac645f7 MW	497	/* --- @dispatch_debug@ --- *
	498	*
	499	* Arguments: @const char *fmt@ = a format string
	500	* @...@ = additional arguments
	501	*
	502	* Returns: ---
	503	*
	504	* Use: Writes a formatted message to standard output if dispatch
	505	* debugging is enabled.
	506	*/
	507
	508	void dispatch_debug(const char *fmt, ...)
	509	{
	510	va_list ap;
	511	const char *e = getenv("CATACOMB_CPUDISPATCH_DEBUG");
	512
	513	if (e && e != 'n' && e != '0') {
	514	va_start(ap, fmt);
	515	fputs("Catacomb CPUDISPATCH: ", stderr);
	516	vfprintf(stderr, fmt, ap);
	517	fputc('\n', stderr);
	518	va_end(ap);
	519	}
	520	}
	521
08e2be29 MW	522	/* --- @check_env@ --- *
	523	*
	524	* Arguments: @const char *ftok@ = feature token
	525	*
	526	* Returns: Zero if the feature is forced off; positive if it's forced
	527	* on; negative if the user hasn't decided.
	528	*
	529	* Use: Checks the environment variable `CATACOMB_CPUFEAT' for the
	530	* feature token @ftok@. The variable, if it exists, should be
	531	* a space-separated sequence of `+tok' and `-tok' items. These
	532	* tokens may end in `*', which matches any suffix.
	533	*/
	534
	535	static int IGNORABLE check_env(const char *ftok)
	536	{
	537	const char p, q, *pp;
	538	int d;
	539
	540	p = getenv("CATACOMB_CPUFEAT");
	541	if (!p) return (-1);
	542
	543	for (;;) {
	544	while (isspace((unsigned char)*p)) p++;
	545	if (!*p) return (-1);
	546	switch (*p) {
	547	case '+': d = +1; p++; break;
	548	case '-': d = 0; p++; break;
	549	default: d = -1; break;
	550	}
	551	for (q = p; q && !isspace((unsigned char)q); q++);
	552	if (d >= 0) {
	553	for (pp = ftok; p < q && pp && p == *pp; p++, pp++);
	554	if ((p == q && !pp) \|\| (p == '*' && p + 1 == q)) return (d);
	555	}
	556	p = q;
	557	}
	558	return (-1);
	559	}
	560
	561	/* --- @cpu_feature_p@ --- *
	562	*
	563	* Arguments: @unsigned feat@ = a @CPUFEAT_...@ code
	564	*
	565	* Returns: Nonzero if the feature is available.
	566	*/
	567
	568	#include <stdio.h>
	569
fac645f7 MW	570	static int IGNORABLE
	571	feat_debug(const char ftok, const char check, int verdict)
	572	{
	573	if (verdict >= 0) {
	574	dispatch_debug("feature `%s': %s -> %s", ftok, check,
	575	verdict ? "available" : "absent");
	576	}
	577	return (verdict);
	578	}
	579
08e2be29 MW	580	int cpu_feature_p(int feat)
	581	{
	582	int IGNORABLE f;
	583	IGNORE(f);
fac645f7 MW	584	#define CASE_CPUFEAT(feat, ftok, cond) case CPUFEAT_##feat: \
	585	if ((f = feat_debug(ftok, "environment override", \
	586	check_env(ftok))) >= 0) \
	587	return (f); \
	588	else \
	589	return (feat_debug(ftok, "runtime probe", cond));
08e2be29 MW	590
08e2be29 MW	591	switch (feat) {
0f23f75f	592	#if CPUFAM_X86 \|\| CPUFAM_AMD64
fac645f7	593	CASE_CPUFEAT(X86_SSE2, "x86:sse2",
4fab22c2 MW	594	cpuid_features_p(CPUID1D_SSE2, 0) &&
4fab22c2 MW	595	xmm_registers_available_p());
fac645f7	596	CASE_CPUFEAT(X86_AESNI, "x86:aesni",
4fab22c2 MW	597	cpuid_features_p(CPUID1D_SSE2, CPUID1C_AESNI) &&
4fab22c2 MW	598	xmm_registers_available_p());
d25653be	599	CASE_CPUFEAT(X86_RDRAND, "x86:rdrand",
35b1eba8	600	cpuid_features_p(0, CPUID1C_RDRAND) && rdrand_works_p());
08e2be29	601	#endif
a02a22d4 MW	602	#ifdef CAPMAP
a02a22d4 MW	603	# define FEATP__CASE(feat, tok) \
0aec0658	604	CASE_CPUFEAT(feat, tok, get_hwcaps() & HF_##feat)
a02a22d4 MW	605	CAPMAP(FEATP__CASE)
	606	#undef FEATP__CASE
	607	#endif
08e2be29	608	default:
fac645f7	609	dispatch_debug("denying unknown feature %d", feat);
08e2be29 MW	610	return (0);
08e2be29 MW	611	}
fac645f7	612	#undef CASE_CPUFEAT
08e2be29 MW	613	}
	614
	615	/----- That's all, folks -------------------------------------------------/