[catacomb] / base / dispatch.c

/* -*-c-*-
 *
 * CPU-specific dispatch
 *
 * (c) 2015 Straylight/Edgeware
 */

/*----- Licensing notice --------------------------------------------------*
 *
 * This file is part of Catacomb.
 *
 * Catacomb is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Library General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * Catacomb is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with Catacomb; if not, write to the Free
 * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
 * MA 02111-1307, USA.
 */

/*----- Header files ------------------------------------------------------*/

#include "config.h"

#include <ctype.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <mLib/macros.h>

#include "dispatch.h"

/*----- Intel x86/AMD64 feature probing -----------------------------------*/

#if CPUFAM_X86 || CPUFAM_AMD64

#  define CPUID1D_SSE2 (1u << 26)
#  define CPUID1D_FXSR (1u << 24)
#  define CPUID1C_PCLMUL (1u << 1)
#  define CPUID1C_SSSE3 (1u << 9)
#  define CPUID1C_AESNI (1u << 25)
#  define CPUID1C_AVX (1u << 28)
#  define CPUID1C_RDRAND (1u << 30)

struct cpuid { unsigned a, b, c, d; };
extern int dispatch_x86ish_cpuid(struct cpuid *, unsigned a, unsigned c);
extern int dispatch_x86ish_xmmregisters_p(void);

static void cpuid(struct cpuid *cc, unsigned a, unsigned c)
{
  int rc = dispatch_x86ish_cpuid(cc, a, c);
  if (rc)
    dispatch_debug("CPUID instruction not available");
  else
    dispatch_debug("CPUID(%08x, %08x) -> %08x, %08x, %08x, %08x",
		   a, c, cc->a, cc->b, cc->c, cc->d);
}

static unsigned cpuid_maxleaf(void)
  { struct cpuid c; cpuid(&c, 0, 0); return (c.a); }

/* --- @cpuid_features_p@ --- *
 *
 * Arguments:	@unsigned dbits@ = bits to check in EDX
 *		@unsigned cbits@ = bits to check in ECX
 *
 * Returns:	Nonzero if all the requested bits are set in the CPUID result
 *		on leaf 1.
 */

static int cpuid_features_p(unsigned dbits, unsigned cbits)
{
  struct cpuid c;
  if (cpuid_maxleaf() < 1) return (0);
  cpuid(&c, 1, 0);
  return ((c.d & dbits) == dbits && (c.c & cbits) == cbits);
}

/* --- @xmm_registers_available_p@ --- *
 *
 * Arguments:	---
 *
 * Returns:	Nonzero if the operating system has made the XMM registers
 *		available for use.
 */

static int xmm_registers_available_p(void)
{
  int f = dispatch_x86ish_xmmregisters_p();

  dispatch_debug("XMM registers %savailable", f ? "" : "not ");
  return (f);
}

#endif

/*----- General feature probing using auxiliary vectors -------------------*/

/* Try to find the system's definitions for auxiliary vector entries. */
#ifdef HAVE_SYS_AUXV_H
#  include <sys/auxv.h>
#endif
#ifdef HAVE_LINUX_AUXVEC_H
#  include <linux/auxvec.h>
#endif
#ifdef HAVE_ASM_HWCAP_H
#  include <asm/hwcap.h>
#endif

/* The type of entries in the auxiliary vector.  I'm assuming that `unsigned
 * long' matches each platform's word length; if this is false then we'll
 * need some host-specific tweaking here.
 */
union auxval { long i; unsigned long u; const void *p; };
struct auxentry { unsigned long type; union auxval value; };

/* Register each CPU family's interest in the auxiliary vector.  Make sure
 * that the necessary entry types are defined.  This is primarily ordered by
 * entry type to minimize duplication.
 */
#if defined(AT_HWCAP) && CPUFAM_ARMEL
#  define WANT_ANY 1
#  define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
#endif

#if defined(AT_HWCAP) && CPUFAM_ARM64
#  define WANT_ANY 1
#  define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
#endif

#if defined(AT_HWCAP2) && CPUFAM_ARMEL
#  define WANT_ANY 1
#  define WANT_AT_HWCAP2(_) _(AT_HWCAP2, u, hwcap2)
#endif

/* If we couldn't find any interesting entries then we can switch all of this
 * machinery off.  Also do that if we have no means for atomic updates.
 */
#if WANT_ANY && CPU_DISPATCH_P

/* The main output of this section is a bitmask of detected features.  The
 * least significant bit will be set if we've tried to probe.  Always access
 * this using `DISPATCH_LOAD' and `DISPATCH_STORE'.
 */
static unsigned hwcaps = 0;

/* For each potentially interesting type which turned out not to exist or be
 * wanted, define a dummy macro for the sake of the next step.
 */
#ifndef WANT_AT_HWCAP
#  define WANT_AT_HWCAP(_)
#endif
#ifndef WANT_AT_HWCAP2
#  define WANT_AT_HWCAP2(_)
#endif

/* For each CPU family, define two lists.
 *
 *   * `WANTAUX' is a list of the `WANT_AT_MUMBLE' macros which the CPU
 *     family tried to register interest in above.  Each entry contains the
 *     interesting auxiliary vector entry type, the name of the union branch
 *     for its value, and the name of the slot in `struct auxprobe' in which
 *     to store the value.
 *
 *   * `CAPMAP' is a list describing the output features which the CPU family
 *     intends to satisfy from the auxiliary vector.  Each entry contains a
 *     feature name suffix, and the token name (for `check_env').
 */
#if CPUFAM_ARMEL
#  define WANTAUX(_)							\
	WANT_AT_HWCAP(_)						\
	WANT_AT_HWCAP2(_)
#  define CAPMAP(_)							\
	_(ARM_VFP, "arm:vfp")						\
	_(ARM_NEON, "arm:neon")						\
	_(ARM_V4, "arm:v4")						\
	_(ARM_D32, "arm:d32")						\
	_(ARM_AES, "arm:aes")						\
	_(ARM_PMULL, "arm:pmull")
#endif
#if CPUFAM_ARM64
#  define WANTAUX(_)							\
	WANT_AT_HWCAP(_)
#  define CAPMAP(_)							\
	_(ARM_AES, "arm:aes")						\
	_(ARM_PMULL, "arm:pmull")
#endif

/* Build the bitmask for `hwcaps' from the `CAPMAP' list. */
enum {
  HFI_PROBED = 0,
#define HFI__ENUM(feat, tok) HFI_##feat,
  CAPMAP(HFI__ENUM)
#undef HFI__ENUM
  HFI__END
};
enum {
  HF_PROBED = 1,
#define HF__FLAG(feat, tok) HF_##feat = 1 << HFI_##feat,
  CAPMAP(HF__FLAG)
#undef HF__FLAG
  HF__END
};

/* Build a structure in which we can capture the interesting data from the
 * auxiliary vector.
 */
#define AUXUTYPE_i long
#define AUXUTYPE_u unsigned long
#define AUXUTYPE_p const void *
struct auxprobe {
#define AUXPROBE__SLOT(type, ubranch, slot) AUXUTYPE_##ubranch slot;
  WANTAUX(AUXPROBE__SLOT)
#undef AUXPROBE_SLOT
};

/* --- @probe_hwcaps@ --- *
 *
 * Arguments:	---
 *
 * Returns:	---
 *
 * Use:		Attempt to find the auxiliary vector (which is well hidden)
 *		and discover interesting features from it.
 */

static void probe_hwcaps(void)
{
  unsigned hw = HF_PROBED;
  struct auxprobe probed = { 0 };

  /* Populate `probed' with the information we manage to retrieve from the
   * auxiliary vector.  Slots we couldn't find are left zero-valued.
   */
#if defined(HAVE_GETAUXVAL)
  /* Shiny new libc lets us request individual entry types.  This is almost
   * too easy.
   */
#  define CAP__GET(type, ubranch, slot)					\
	probed.slot = (AUXUTYPE_##ubranch)getauxval(type);
  WANTAUX(CAP__GET)
#else
  /* Otherwise we're a bit stuck, really.  Modern Linux kernels make a copy
   * of the vector available in `/procc' so we could try that.
   *
   * The usual place is stuck on the end of the environment vector, but that
   * may well have moved, and we have no way of telling whether it has or
   * whether there was ever an auxiliary vector there at all; so don't do
   * that.
   */
  {
    FILE *fp = 0;
    unsigned char *p = 0, *q = 0;
    const struct auxentry *a;
    size_t sz, off, n;

    /* Open the file and read it into a memory chunk. */
    if ((fp = fopen("/proc/self/auxv", "rb")) == 0) goto clean;
    sz = 4096; off = 0;
    if ((p = malloc(sz)) == 0) goto clean;
    for (;;) {
      n = fread(p + off, 1, sz - off, fp);
      off += n;
      if (off < sz) break;
      sz *= 2; if ((q = realloc(p, sz)) == 0) break;
      p = q;
    }

    /* Work through the vector (or as much of it as we found) and extract the
     * types we're interested in.
     */
    for (a = (const struct auxentry *)p,
	   n = sz/sizeof(struct auxentry);
	 n--; a++) {
      switch (a->type) {
#define CAP__SWITCH(type, ubranch, slot)				\
	case type: probed.slot = a->value.ubranch; break;
	WANTAUX(CAP__SWITCH)
	case AT_NULL: goto clean;
      }
    }

  clean:
    if (p) free(p);
    if (fp) fclose(fp);
  }
#endif

  /* Each CPU family now has to pick through what was found and stashed in
   * `probed', and set the appropriate flag bits in `hw'.
   */
#if CPUFAM_ARMEL
  if (probed.hwcap & HWCAP_VFPv3) hw |= HF_ARM_VFP;
  if (probed.hwcap & HWCAP_NEON) hw |= HF_ARM_NEON;
  if (probed.hwcap & HWCAP_VFPD32) hw |= HF_ARM_D32;
  if (probed.hwcap & HWCAP_VFPv4) hw |= HF_ARM_V4;
#  ifdef HWCAP2_AES
  if (probed.hwcap2 & HWCAP2_AES) hw |= HF_ARM_AES;
#  endif
#  ifdef HWCAP2_PMULL
  if (probed.hwcap2 & HWCAP2_PMULL) hw |= HF_ARM_PMULL;
#  endif
#endif
#if CPUFAM_ARM64
  if (probed.hwcap & HWCAP_AES) hw |= HF_ARM_AES;
  if (probed.hwcap & HWCAP_PMULL) hw |= HF_ARM_PMULL;
#endif

  /* Store the bitmask of features we probed for everyone to see. */
  DISPATCH_STORE(hwcaps, hw);

  /* Finally, make a report about the things we found.  (Doing this earlier
   * will pointlessly widen the window in which multiple threads will do the
   * above auxiliary-vector probing.)
   */
#define CAP__DEBUG(feat, tok)						\
  dispatch_debug("check auxv for feature `%s': %s", tok,		\
		 hw & HF_##feat ? "available" : "absent");
  CAPMAP(CAP__DEBUG)
#undef CAP__DEBUG
}

/* --- @get_hwcaps@ --- *
 *
 * Arguments:	---
 *
 * Returns:	A mask of hardware capabilities and other features, as probed
 *		from the auxiliary vector.
 */

static unsigned get_hwcaps(void)
{
  unsigned hw;

  DISPATCH_LOAD(hwcaps, hw);
  if (!(hwcaps & HF_PROBED)) { probe_hwcaps(); DISPATCH_LOAD(hwcaps, hw); }
  return (hw);
}

#endif

/*----- External interface ------------------------------------------------*/

/* --- @dispatch_debug@ --- *
 *
 * Arguments:	@const char *fmt@ = a format string
 *		@...@ = additional arguments
 *
 * Returns:	---
 *
 * Use:		Writes a formatted message to standard output if dispatch
 *		debugging is enabled.
 */

void dispatch_debug(const char *fmt, ...)
{
  va_list ap;
  const char *e = getenv("CATACOMB_CPUDISPATCH_DEBUG");

  if (e && *e != 'n' && *e != '0') {
    va_start(ap, fmt);
    fputs("Catacomb CPUDISPATCH: ", stderr);
    vfprintf(stderr, fmt, ap);
    fputc('\n', stderr);
    va_end(ap);
  }
}

/* --- @check_env@ --- *
 *
 * Arguments:	@const char *ftok@ = feature token
 *
 * Returns:	Zero if the feature is forced off; positive if it's forced
 *		on; negative if the user hasn't decided.
 *
 * Use:		Checks the environment variable `CATACOMB_CPUFEAT' for the
 *		feature token @ftok@.  The variable, if it exists, should be
 *		a space-separated sequence of `+tok' and `-tok' items.  These
 *		tokens may end in `*', which matches any suffix.
 */

static int IGNORABLE check_env(const char *ftok)
{
  const char *p, *q, *pp;
  int d;

  p = getenv("CATACOMB_CPUFEAT");
  if (!p) return (-1);

  for (;;) {
    while (isspace((unsigned char)*p)) p++;
    if (!*p) return (-1);
    switch (*p) {
      case '+': d = +1; p++; break;
      case '-': d =  0; p++; break;
      default:  d = -1;      break;
    }
    for (q = p; *q && !isspace((unsigned char)*q); q++);
    if (d >= 0) {
      for (pp = ftok; p < q && *pp && *p == *pp; p++, pp++);
      if ((p == q && !*pp) || (*p == '*' && p + 1 == q)) return (d);
    }
    p = q;
  }
  return (-1);
}

/* --- @cpu_feature_p@ --- *
 *
 * Arguments:	@unsigned feat@ = a @CPUFEAT_...@ code
 *
 * Returns:	Nonzero if the feature is available.
 */

#include <stdio.h>

static int IGNORABLE
  feat_debug(const char *ftok, const char *check, int verdict)
{
  if (verdict >= 0) {
    dispatch_debug("feature `%s': %s -> %s", ftok, check,
		   verdict ? "available" : "absent");
  }
  return (verdict);
}

int cpu_feature_p(int feat)
{
  int IGNORABLE f;
  IGNORE(f);
#define CASE_CPUFEAT(feat, ftok, cond) case CPUFEAT_##feat:		\
  if ((f = feat_debug(ftok, "environment override",			\
		      check_env(ftok))) >= 0)				\
    return (f);								\
  else									\
    return (feat_debug(ftok, "runtime probe", cond));

  switch (feat) {
#if CPUFAM_X86 || CPUFAM_AMD64
    CASE_CPUFEAT(X86_SSE2, "x86:sse2",
		 cpuid_features_p(CPUID1D_SSE2, 0) &&
		 xmm_registers_available_p());
    CASE_CPUFEAT(X86_AESNI, "x86:aesni",
		 cpuid_features_p(CPUID1D_SSE2, CPUID1C_AESNI) &&
		 xmm_registers_available_p());
    CASE_CPUFEAT(X86_RDRAND, "x86:rdrand",
		 cpuid_features_p(0, CPUID1C_RDRAND));
    CASE_CPUFEAT(X86_AVX, "x86:avx",
		 cpuid_features_p(0, CPUID1C_AVX) &&
		 xmm_registers_available_p());
    CASE_CPUFEAT(X86_SSSE3, "x86:ssse3",
		 cpuid_features_p(0, CPUID1C_SSSE3) &&
		 xmm_registers_available_p());
    CASE_CPUFEAT(X86_PCLMUL, "x86:pclmul",
		 cpuid_features_p(0, CPUID1C_PCLMUL) &&
		 xmm_registers_available_p());
#endif
#ifdef CAPMAP
#  define FEATP__CASE(feat, tok)					\
	CASE_CPUFEAT(feat, tok,	get_hwcaps() & HF_##feat)
    CAPMAP(FEATP__CASE)
#undef FEATP__CASE
#endif
    default:
      dispatch_debug("denying unknown feature %d", feat);
      return (0);
  }
#undef CASE_CPUFEAT
}

/*----- That's all, folks -------------------------------------------------*/
Commit	Line	Data
08e2be29 MW	1	/* --c--
	2	*
	3	* CPU-specific dispatch
	4	*
	5	* (c) 2015 Straylight/Edgeware
	6	*/
	7
	8	/----- Licensing notice --------------------------------------------------
	9	*
	10	* This file is part of Catacomb.
	11	*
	12	* Catacomb is free software; you can redistribute it and/or modify
	13	* it under the terms of the GNU Library General Public License as
	14	* published by the Free Software Foundation; either version 2 of the
	15	* License, or (at your option) any later version.
	16	*
	17	* Catacomb is distributed in the hope that it will be useful,
	18	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	20	* GNU Library General Public License for more details.
	21	*
	22	* You should have received a copy of the GNU Library General Public
	23	* License along with Catacomb; if not, write to the Free
	24	* Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
	25	* MA 02111-1307, USA.
	26	*/
	27
	28	/----- Header files ------------------------------------------------------/
	29
	30	#include "config.h"
	31
	32	#include <ctype.h>
fac645f7 MW	33	#include <stdarg.h>
fac645f7 MW	34	#include <stdio.h>
08e2be29 MW	35	#include <stdlib.h>
	36	#include <string.h>
	37
	38	#include <mLib/macros.h>
	39
	40	#include "dispatch.h"
	41
d26ad211	42	/----- Intel x86/AMD64 feature probing -----------------------------------/
08e2be29	43
0f23f75f	44	#if CPUFAM_X86 \|\| CPUFAM_AMD64
08e2be29	45
7680f863 MW	46	# define CPUID1D_SSE2 (1u << 26)
7680f863 MW	47	# define CPUID1D_FXSR (1u << 24)
9e6a4409 MW	48	# define CPUID1C_PCLMUL (1u << 1)
9e6a4409 MW	49	# define CPUID1C_SSSE3 (1u << 9)
7680f863	50	# define CPUID1C_AESNI (1u << 25)
b9b279b4	51	# define CPUID1C_AVX (1u << 28)
d25653be	52	# define CPUID1C_RDRAND (1u << 30)
08e2be29 MW	53
08e2be29 MW	54	struct cpuid { unsigned a, b, c, d; };
a3ad4421 MW	55	extern int dispatch_x86ish_cpuid(struct cpuid *, unsigned a, unsigned c);
a3ad4421 MW	56	extern int dispatch_x86ish_xmmregisters_p(void);
08e2be29 MW	57
	58	static void cpuid(struct cpuid *cc, unsigned a, unsigned c)
	59	{
a3ad4421 MW	60	int rc = dispatch_x86ish_cpuid(cc, a, c);
a3ad4421 MW	61	if (rc)
fac645f7	62	dispatch_debug("CPUID instruction not available");
a3ad4421 MW	63	else
	64	dispatch_debug("CPUID(%08x, %08x) -> %08x, %08x, %08x, %08x",
	65	a, c, cc->a, cc->b, cc->c, cc->d);
08e2be29 MW	66	}
	67
	68	static unsigned cpuid_maxleaf(void)
	69	{ struct cpuid c; cpuid(&c, 0, 0); return (c.a); }
	70
11b17977 MW	71	/* --- @cpuid_features_p@ --- *
	72	*
	73	* Arguments: @unsigned dbits@ = bits to check in EDX
	74	* @unsigned cbits@ = bits to check in ECX
	75	*
	76	* Returns: Nonzero if all the requested bits are set in the CPUID result
	77	* on leaf 1.
	78	*/
	79
08e2be29 MW	80	static int cpuid_features_p(unsigned dbits, unsigned cbits)
	81	{
	82	struct cpuid c;
	83	if (cpuid_maxleaf() < 1) return (0);
	84	cpuid(&c, 1, 0);
	85	return ((c.d & dbits) == dbits && (c.c & cbits) == cbits);
	86	}
	87
11b17977 MW	88	/* --- @xmm_registers_available_p@ --- *
	89	*
	90	* Arguments: ---
	91	*
	92	* Returns: Nonzero if the operating system has made the XMM registers
	93	* available for use.
	94	*/
	95
acbe16df MW	96	static int xmm_registers_available_p(void)
acbe16df MW	97	{
a3ad4421 MW	98	int f = dispatch_x86ish_xmmregisters_p();
a3ad4421 MW	99
0f23f75f	100	dispatch_debug("XMM registers %savailable", f ? "" : "not ");
acbe16df	101	return (f);
acbe16df MW	102	}
acbe16df MW	103
08e2be29 MW	104	#endif
08e2be29 MW	105
a02a22d4 MW	106	/----- General feature probing using auxiliary vectors -------------------/
	107
	108	/* Try to find the system's definitions for auxiliary vector entries. */
	109	#ifdef HAVE_SYS_AUXV_H
	110	# include <sys/auxv.h>
a0e9bb8a MW	111	#endif
	112	#ifdef HAVE_LINUX_AUXVEC_H
	113	# include <linux/auxvec.h>
	114	#endif
	115	#ifdef HAVE_ASM_HWCAP_H
	116	# include <asm/hwcap.h>
a02a22d4 MW	117	#endif
	118
	119	/* The type of entries in the auxiliary vector. I'm assuming that `unsigned
	120	* long' matches each platform's word length; if this is false then we'll
	121	* need some host-specific tweaking here.
	122	*/
	123	union auxval { long i; unsigned long u; const void *p; };
	124	struct auxentry { unsigned long type; union auxval value; };
	125
	126	/* Register each CPU family's interest in the auxiliary vector. Make sure
	127	* that the necessary entry types are defined. This is primarily ordered by
	128	* entry type to minimize duplication.
	129	*/
61bd904b MW	130	#if defined(AT_HWCAP) && CPUFAM_ARMEL
	131	# define WANT_ANY 1
	132	# define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
	133	#endif
a02a22d4	134
e492db88 MW	135	#if defined(AT_HWCAP) && CPUFAM_ARM64
	136	# define WANT_ANY 1
	137	# define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
	138	#endif
	139
26e182fc MW	140	#if defined(AT_HWCAP2) && CPUFAM_ARMEL
	141	# define WANT_ANY 1
	142	# define WANT_AT_HWCAP2(_) _(AT_HWCAP2, u, hwcap2)
	143	#endif
	144
a02a22d4 MW	145	/* If we couldn't find any interesting entries then we can switch all of this
	146	* machinery off. Also do that if we have no means for atomic updates.
	147	*/
	148	#if WANT_ANY && CPU_DISPATCH_P
	149
	150	/* The main output of this section is a bitmask of detected features. The
	151	* least significant bit will be set if we've tried to probe. Always access
	152	* this using `DISPATCH_LOAD' and `DISPATCH_STORE'.
	153	*/
	154	static unsigned hwcaps = 0;
	155
	156	/* For each potentially interesting type which turned out not to exist or be
	157	* wanted, define a dummy macro for the sake of the next step.
	158	*/
	159	#ifndef WANT_AT_HWCAP
	160	# define WANT_AT_HWCAP(_)
	161	#endif
f013a4f2 MW	162	#ifndef WANT_AT_HWCAP2
	163	# define WANT_AT_HWCAP2(_)
	164	#endif
a02a22d4 MW	165
	166	/* For each CPU family, define two lists.
	167	*
	168	* * `WANTAUX' is a list of the `WANT_AT_MUMBLE' macros which the CPU
	169	* family tried to register interest in above. Each entry contains the
	170	* interesting auxiliary vector entry type, the name of the union branch
	171	* for its value, and the name of the slot in `struct auxprobe' in which
	172	* to store the value.
	173	*
	174	* * `CAPMAP' is a list describing the output features which the CPU family
	175	* intends to satisfy from the auxiliary vector. Each entry contains a
	176	* feature name suffix, and the token name (for `check_env').
	177	*/
61bd904b MW	178	#if CPUFAM_ARMEL
61bd904b MW	179	# define WANTAUX(_) \
26e182fc MW	180	WANT_AT_HWCAP(_) \
26e182fc MW	181	WANT_AT_HWCAP2(_)
61bd904b MW	182	# define CAPMAP(_) \
	183	_(ARM_VFP, "arm:vfp") \
	184	_(ARM_NEON, "arm:neon") \
	185	_(ARM_V4, "arm:v4") \
26e182fc	186	_(ARM_D32, "arm:d32") \
9e6a4409 MW	187	_(ARM_AES, "arm:aes") \
9e6a4409 MW	188	_(ARM_PMULL, "arm:pmull")
61bd904b	189	#endif
e492db88 MW	190	#if CPUFAM_ARM64
	191	# define WANTAUX(_) \
	192	WANT_AT_HWCAP(_)
	193	# define CAPMAP(_) \
9e6a4409 MW	194	_(ARM_AES, "arm:aes") \
9e6a4409 MW	195	_(ARM_PMULL, "arm:pmull")
e492db88	196	#endif
a02a22d4 MW	197
	198	/* Build the bitmask for `hwcaps' from the `CAPMAP' list. */
	199	enum {
	200	HFI_PROBED = 0,
	201	#define HFI__ENUM(feat, tok) HFI_##feat,
	202	CAPMAP(HFI__ENUM)
	203	#undef HFI__ENUM
	204	HFI__END
	205	};
	206	enum {
	207	HF_PROBED = 1,
	208	#define HF__FLAG(feat, tok) HF_##feat = 1 << HFI_##feat,
	209	CAPMAP(HF__FLAG)
	210	#undef HF__FLAG
	211	HF__END
	212	};
	213
	214	/* Build a structure in which we can capture the interesting data from the
	215	* auxiliary vector.
	216	*/
	217	#define AUXUTYPE_i long
	218	#define AUXUTYPE_u unsigned long
	219	#define AUXUTYPE_p const void *
	220	struct auxprobe {
	221	#define AUXPROBE__SLOT(type, ubranch, slot) AUXUTYPE_##ubranch slot;
	222	WANTAUX(AUXPROBE__SLOT)
	223	#undef AUXPROBE_SLOT
	224	};
	225
	226	/* --- @probe_hwcaps@ --- *
	227	*
	228	* Arguments: ---
	229	*
	230	* Returns: ---
	231	*
	232	* Use: Attempt to find the auxiliary vector (which is well hidden)
	233	* and discover interesting features from it.
	234	*/
	235
	236	static void probe_hwcaps(void)
	237	{
	238	unsigned hw = HF_PROBED;
	239	struct auxprobe probed = { 0 };
	240
	241	/* Populate `probed' with the information we manage to retrieve from the
	242	* auxiliary vector. Slots we couldn't find are left zero-valued.
	243	*/
	244	#if defined(HAVE_GETAUXVAL)
	245	/* Shiny new libc lets us request individual entry types. This is almost
	246	* too easy.
	247	*/
84850eec MW	248	# define CAP__GET(type, ubranch, slot) \
84850eec MW	249	probed.slot = (AUXUTYPE_##ubranch)getauxval(type);
a02a22d4 MW	250	WANTAUX(CAP__GET)
	251	#else
	252	/* Otherwise we're a bit stuck, really. Modern Linux kernels make a copy
	253	* of the vector available in `/procc' so we could try that.
	254	*
	255	* The usual place is stuck on the end of the environment vector, but that
	256	* may well have moved, and we have no way of telling whether it has or
	257	* whether there was ever an auxiliary vector there at all; so don't do
	258	* that.
	259	*/
	260	{
	261	FILE *fp = 0;
	262	unsigned char p = 0, q = 0;
	263	const struct auxentry *a;
	264	size_t sz, off, n;
	265
	266	/* Open the file and read it into a memory chunk. */
	267	if ((fp = fopen("/proc/self/auxv", "rb")) == 0) goto clean;
	268	sz = 4096; off = 0;
	269	if ((p = malloc(sz)) == 0) goto clean;
	270	for (;;) {
	271	n = fread(p + off, 1, sz - off, fp);
	272	off += n;
	273	if (off < sz) break;
	274	sz *= 2; if ((q = realloc(p, sz)) == 0) break;
	275	p = q;
	276	}
	277
	278	/* Work through the vector (or as much of it as we found) and extract the
	279	* types we're interested in.
	280	*/
	281	for (a = (const struct auxentry *)p,
	282	n = sz/sizeof(struct auxentry);
	283	n--; a++) {
	284	switch (a->type) {
	285	#define CAP__SWITCH(type, ubranch, slot) \
	286	case type: probed.slot = a->value.ubranch; break;
	287	WANTAUX(CAP__SWITCH)
dfcb2a0b	288	case AT_NULL: goto clean;
a02a22d4 MW	289	}
	290	}
	291
	292	clean:
	293	if (p) free(p);
	294	if (fp) fclose(fp);
	295	}
	296	#endif
	297
	298	/* Each CPU family now has to pick through what was found and stashed in
	299	* `probed', and set the appropriate flag bits in `hw'.
	300	*/
61bd904b MW	301	#if CPUFAM_ARMEL
	302	if (probed.hwcap & HWCAP_VFPv3) hw \|= HF_ARM_VFP;
	303	if (probed.hwcap & HWCAP_NEON) hw \|= HF_ARM_NEON;
	304	if (probed.hwcap & HWCAP_VFPD32) hw \|= HF_ARM_D32;
	305	if (probed.hwcap & HWCAP_VFPv4) hw \|= HF_ARM_V4;
26e182fc MW	306	# ifdef HWCAP2_AES
	307	if (probed.hwcap2 & HWCAP2_AES) hw \|= HF_ARM_AES;
	308	# endif
9e6a4409 MW	309	# ifdef HWCAP2_PMULL
	310	if (probed.hwcap2 & HWCAP2_PMULL) hw \|= HF_ARM_PMULL;
	311	# endif
61bd904b	312	#endif
e492db88 MW	313	#if CPUFAM_ARM64
e492db88 MW	314	if (probed.hwcap & HWCAP_AES) hw \|= HF_ARM_AES;
9e6a4409	315	if (probed.hwcap & HWCAP_PMULL) hw \|= HF_ARM_PMULL;
e492db88	316	#endif
a02a22d4 MW	317
	318	/* Store the bitmask of features we probed for everyone to see. */
	319	DISPATCH_STORE(hwcaps, hw);
	320
	321	/* Finally, make a report about the things we found. (Doing this earlier
	322	* will pointlessly widen the window in which multiple threads will do the
	323	* above auxiliary-vector probing.)
	324	*/
	325	#define CAP__DEBUG(feat, tok) \
	326	dispatch_debug("check auxv for feature `%s': %s", tok, \
	327	hw & HF_##feat ? "available" : "absent");
	328	CAPMAP(CAP__DEBUG)
	329	#undef CAP__DEBUG
	330	}
	331
	332	/* --- @get_hwcaps@ --- *
	333	*
	334	* Arguments: ---
	335	*
	336	* Returns: A mask of hardware capabilities and other features, as probed
	337	* from the auxiliary vector.
	338	*/
	339
	340	static unsigned get_hwcaps(void)
	341	{
	342	unsigned hw;
	343
	344	DISPATCH_LOAD(hwcaps, hw);
	345	if (!(hwcaps & HF_PROBED)) { probe_hwcaps(); DISPATCH_LOAD(hwcaps, hw); }
	346	return (hw);
	347	}
	348
	349	#endif
	350
d26ad211 MW	351	/----- External interface ------------------------------------------------/
d26ad211 MW	352
fac645f7 MW	353	/* --- @dispatch_debug@ --- *
	354	*
	355	* Arguments: @const char *fmt@ = a format string
	356	* @...@ = additional arguments
	357	*
	358	* Returns: ---
	359	*
	360	* Use: Writes a formatted message to standard output if dispatch
	361	* debugging is enabled.
	362	*/
	363
	364	void dispatch_debug(const char *fmt, ...)
	365	{
	366	va_list ap;
	367	const char *e = getenv("CATACOMB_CPUDISPATCH_DEBUG");
	368
	369	if (e && e != 'n' && e != '0') {
	370	va_start(ap, fmt);
	371	fputs("Catacomb CPUDISPATCH: ", stderr);
	372	vfprintf(stderr, fmt, ap);
	373	fputc('\n', stderr);
	374	va_end(ap);
	375	}
	376	}
	377
08e2be29 MW	378	/* --- @check_env@ --- *
	379	*
	380	* Arguments: @const char *ftok@ = feature token
	381	*
	382	* Returns: Zero if the feature is forced off; positive if it's forced
	383	* on; negative if the user hasn't decided.
	384	*
	385	* Use: Checks the environment variable `CATACOMB_CPUFEAT' for the
	386	* feature token @ftok@. The variable, if it exists, should be
	387	* a space-separated sequence of `+tok' and `-tok' items. These
	388	* tokens may end in `*', which matches any suffix.
	389	*/
	390
	391	static int IGNORABLE check_env(const char *ftok)
	392	{
	393	const char p, q, *pp;
	394	int d;
	395
	396	p = getenv("CATACOMB_CPUFEAT");
	397	if (!p) return (-1);
	398
	399	for (;;) {
	400	while (isspace((unsigned char)*p)) p++;
	401	if (!*p) return (-1);
	402	switch (*p) {
	403	case '+': d = +1; p++; break;
	404	case '-': d = 0; p++; break;
	405	default: d = -1; break;
	406	}
	407	for (q = p; q && !isspace((unsigned char)q); q++);
	408	if (d >= 0) {
	409	for (pp = ftok; p < q && pp && p == *pp; p++, pp++);
	410	if ((p == q && !pp) \|\| (p == '*' && p + 1 == q)) return (d);
	411	}
	412	p = q;
	413	}
	414	return (-1);
	415	}
	416
	417	/* --- @cpu_feature_p@ --- *
	418	*
	419	* Arguments: @unsigned feat@ = a @CPUFEAT_...@ code
	420	*
	421	* Returns: Nonzero if the feature is available.
	422	*/
	423
	424	#include <stdio.h>
	425
fac645f7 MW	426	static int IGNORABLE
	427	feat_debug(const char ftok, const char check, int verdict)
	428	{
	429	if (verdict >= 0) {
	430	dispatch_debug("feature `%s': %s -> %s", ftok, check,
	431	verdict ? "available" : "absent");
	432	}
	433	return (verdict);
	434	}
	435
08e2be29 MW	436	int cpu_feature_p(int feat)
	437	{
	438	int IGNORABLE f;
	439	IGNORE(f);
fac645f7 MW	440	#define CASE_CPUFEAT(feat, ftok, cond) case CPUFEAT_##feat: \
	441	if ((f = feat_debug(ftok, "environment override", \
	442	check_env(ftok))) >= 0) \
	443	return (f); \
	444	else \
	445	return (feat_debug(ftok, "runtime probe", cond));
08e2be29 MW	446
08e2be29 MW	447	switch (feat) {
0f23f75f	448	#if CPUFAM_X86 \|\| CPUFAM_AMD64
fac645f7	449	CASE_CPUFEAT(X86_SSE2, "x86:sse2",
4fab22c2 MW	450	cpuid_features_p(CPUID1D_SSE2, 0) &&
4fab22c2 MW	451	xmm_registers_available_p());
fac645f7	452	CASE_CPUFEAT(X86_AESNI, "x86:aesni",
4fab22c2 MW	453	cpuid_features_p(CPUID1D_SSE2, CPUID1C_AESNI) &&
4fab22c2 MW	454	xmm_registers_available_p());
d25653be MW	455	CASE_CPUFEAT(X86_RDRAND, "x86:rdrand",
d25653be MW	456	cpuid_features_p(0, CPUID1C_RDRAND));
b9b279b4	457	CASE_CPUFEAT(X86_AVX, "x86:avx",
6af2607b MW	458	cpuid_features_p(0, CPUID1C_AVX) &&
6af2607b MW	459	xmm_registers_available_p());
9e6a4409	460	CASE_CPUFEAT(X86_SSSE3, "x86:ssse3",
6af2607b MW	461	cpuid_features_p(0, CPUID1C_SSSE3) &&
6af2607b MW	462	xmm_registers_available_p());
9e6a4409	463	CASE_CPUFEAT(X86_PCLMUL, "x86:pclmul",
6af2607b MW	464	cpuid_features_p(0, CPUID1C_PCLMUL) &&
6af2607b MW	465	xmm_registers_available_p());
08e2be29	466	#endif
a02a22d4 MW	467	#ifdef CAPMAP
a02a22d4 MW	468	# define FEATP__CASE(feat, tok) \
0aec0658	469	CASE_CPUFEAT(feat, tok, get_hwcaps() & HF_##feat)
a02a22d4 MW	470	CAPMAP(FEATP__CASE)
	471	#undef FEATP__CASE
	472	#endif
08e2be29	473	default:
fac645f7	474	dispatch_debug("denying unknown feature %d", feat);
08e2be29 MW	475	return (0);
08e2be29 MW	476	}
fac645f7	477	#undef CASE_CPUFEAT
08e2be29 MW	478	}
	479
	480	/----- That's all, folks -------------------------------------------------/