mdw@git.distorted.org.uk Git - mLib/blob - test/tvec.3.in

   1 .\" -*-nroff-*-
   2 .\"
   3 .\" Manual for the test vector framework
   4 .\"
   5 .\" (c) 2024 Straylight/Edgeware
   6 .\"
   7 .
   8 .\"----- Licensing notice ---------------------------------------------------
   9 .\"
  10 .\" This file is part of the mLib utilities library.
  11 .\"
  12 .\" mLib is free software: you can redistribute it and/or modify it under
  13 .\" the terms of the GNU Library General Public License as published by
  14 .\" the Free Software Foundation; either version 2 of the License, or (at
  15 .\" your option) any later version.
  16 .\"
  17 .\" mLib is distributed in the hope that it will be useful, but WITHOUT
  18 .\" ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  19 .\" FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
  20 .\" License for more details.
  21 .\"
  22 .\" You should have received a copy of the GNU Library General Public
  23 .\" License along with mLib.  If not, write to the Free Software
  24 .\" Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
  25 .\" USA.
  26 .
  27 .\"--------------------------------------------------------------------------
  28 .so ../defs.man \" @@@PRE@@@
  29 .
  30 .\"--------------------------------------------------------------------------
  31 .TH tvec 3mLib "11 March 2024" "Straylight/Edgeware" "mLib utilities library"
  32 .
  33 .\"--------------------------------------------------------------------------
  34 .SH NAME
  35 tvec \- test vector framework
  36 .\" @tvec_begin
  37 .\" @tvec_end
  38 .\" @tvec_read
  39 .\" @tvec_humanoutput
  40 .\" @tvec_tapoutput
  41 .\" @tvec_dfltoutput
  42 .
  43 .\"--------------------------------------------------------------------------
  44 .SH SYNOPSIS
  45 .nf
  46 .B "#include <mLib/tvec.h>"
  47 .PP
  48 .ta 2n
  49 .B "union tvec_misc {"
  50 .B "    const void *p;"
  51 .B "    long i;"
  52 .B "    unsigned long u;"
  53 .B "    double f;"
  54 .B "};"
  55 .B "enum {"
  56 .B "    TVMISC_PTR,"
  57 .B "    TVMISC_INT,"
  58 .B "    TVMISC_UINT,"
  59 .B "    TVMISC_FLT,"
  60 .B "    ...,"
  61 .B "    TVMISC_LIMIT,"
  62 .B "};"
  63 .PP
  64 .ta 2n +2n
  65 .B "union tvec_regval {"
  66 .B "    long i;"
  67 .B "    unsigned long u;"
  68 .B "    void *p;"
  69 .B "    double f;"
  70 .B "    struct { char *p; size_t sz; } text;"
  71 .B "    struct { unsigned char *p; size_t sz; } bytes;"
  72 .B "    struct {"
  73 .B "            unsigned char *p; size_t sz;"
  74 .B "            size_t a, m;"
  75 .B "            size_t off;"
  76 .B "    } buf;"
  77 .B "    TVEC_REGSLOTS"
  78 .B "};"
  79 .B "struct tvec_reg {"
  80 .B "    unsigned f;"
  81 .B "    union tvec_regval v;"
  82 .B "};"
  83 .B "#define TVRF_LIVE ..."
  84 .PP
  85 .ta 2n
  86 .B "struct tvec_regdef {"
  87 .B "    const char *name;"
  88 .B "    const struct tvec_regty *ty;"
  89 .B "    unsigned i;"
  90 .B "    unsigned f;"
  91 .B "    union tvec_misc arg;"
  92 .B "};"
  93 .B "#define TVRF_UNSET ..."
  94 .B "#define TVRF_OPT ..."
  95 .B "#define TVRF_ID ..."
  96 .B "#define TVEC_ENDREGS ..."
  97 .PP
  98 .B "struct tvec_state;"
  99 .PP
 100 .B "struct tvec_env;"
 101 .ta \w'\fBtypedef void tvec_testfn('u
 102 .BI "typedef void tvec_testfn(const struct tvec_reg *" in ,
 103 .BI "   struct tvec_reg *" out ,
 104 .BI "   void *" ctx );
 105 .ta 2n
 106 .B "struct tvec_test {"
 107 .B "    const char *name;"
 108 .B "    const struct tvec_regdef *regs;"
 109 .B "    const struct tvec_env *env;"
 110 .B "    tvec_testfn *fn;"
 111 .B "};"
 112 .B "#define TVEC_ENDTESTS ..."
 113 .PP
 114 .ta 2n
 115 .B "struct tvec_config {"
 116 .B "    const struct tvec_test *tests;"
 117 .B "    unsigned nrout, nreg;"
 118 .B "    size_t regsz;"
 119 .B "};"
 120 .B "struct tvec_output;"
 121 .PP
 122 .ta \w'\fBvoid tvec_begin('u
 123 .BI "void tvec_begin(struct tvec_state *" tv_out ,
 124 .BI "   const struct tvec_config *" config ,
 125 .BI "   struct tvec_output *" o );
 126 .BI "int tvec_end(struct tvec_state *" tv );
 127 .BI "int tvec_read(struct tvec_state *" tv ", const char *" infile ", FILE *" fp );
 128 .PP
 129 .BI "extern struct tvec_output *tvec_humanoutput(FILE *" fp );
 130 .BI "extern struct tvec_output *tvec_tapoutput(FILE *" fp );
 131 .BI "extern struct tvec_output *tvec_dfltoutput(FILE *" fp );
 132 .fi
 133 .
 134 .\"--------------------------------------------------------------------------
 135 .SH DESCRIPTION
 136 .
 137 The
 138 .B <mLib/tvec.h>
 139 header file provides definitions and declarations
 140 for the core of mLib's
 141 .IR "test vector framework" .
 142 .PP
 143 The test vector framework is rather large and complicated,
 144 so the documentation for it is split into multiple manual pages.
 145 This one provides a conceptual overview
 146 and describes the essentials for using it to build simple tests.
 147 .
 148 .SS Conceptual overview
 149 A
 150 .I "test session"
 151 runs a collection of tests
 152 and reports on the outcome.
 153 .PP
 154 A
 155 .I test
 156 involves exercising some functionality
 157 and checking that it behaves properly.
 158 A test can have four
 159 .IR outcomes .
 160 It can
 161 .IR pass :
 162 the functionality behaved properly.
 163 It can
 164 .IR fail :
 165 the functionality did not behave properly.
 166 It can experience an
 167 .IR "expected failure" :
 168 the functionality behaved as expected,
 169 but the expected behaviour is known to be incorrect.
 170 Or it can be
 171 .IR skipped :
 172 for some reason, the test couldn't be performed.
 173 .PP
 174 Tests are gathered together into
 175 .IR "test groups" .
 176 Each test group has a name.
 177 Like a individual tests, test groups also have outcomes:
 178 they can pass, fail, or be skipped.
 179 A test group cannot experience expected failure.
 180 .PP
 181 A session may also encounter
 182 .IR errors ,
 183 e.g., as a result of malformed input
 184 or failures reported by system facilities.
 185 .PP
 186 A test session can either
 187 be driven from data provided by an input file,
 188 or it can be driven by the program alone.
 189 The latter case is called
 190 .I "ad-hoc testing",
 191 and is described in
 192 .BR tvec-adhoc (3).
 193 This manual page describes file-driven testing.
 194 .PP
 195 When it begins a session for file-directed testing,
 196 the program provides a table of
 197 .IR "test definitions" .
 198 A test definition has a
 199 .IR name ,
 200 and also specifies a
 201 .IR "test function" ,
 202 a
 203 .IR "test environment" ,
 204 and a table of
 205 .IR "register definitions" .
 206 Test environments are explained further below.
 207 .PP
 208 A
 209 .I register
 210 is a place which can store a single item of test data;
 211 registers are the means
 212 by which input test data is provided to a test function,
 213 and by which a test function returns its results.
 214 A test definition's register definitions
 215 establish a collection of
 216 .I active
 217 registers.
 218 Each active register has a
 219 .IR name ,
 220 an
 221 .IR index ,
 222 and a
 223 .IR type ,
 224 which are established by its register definition.
 225 The register's name is used to refer to the register in the test data file,
 226 and its index is used to refer to it
 227 in the test function and test environments.
 228 The register's type describes the acceptable values for the register,
 229 and how they are to be compared,
 230 read from the input file,
 231 and dumped in diagnostic output.
 232 New register types can be defined fairly easily: see
 233 .BR tvec_tyimpl (3)
 234 for the details.
 235 A register definition may describe an
 236 .I input
 237 register or an
 238 .I output
 239 register:
 240 input registers provide input data to the test function, while
 241 output registers collect output data from the test function.
 242 The data file provides values for both input and output registers:
 243 the values for the input registers are passed to the test function;
 244 the values for the output registers are
 245 .I "reference outputs"
 246 against which the test function's outputs are to be checked.
 247 .PP
 248 The test function is called with two vectors of registers,
 249 one containing input values for the test function to read,
 250 and another for output values that the test function should write;
 251 and a
 252 .I context
 253 provided by the test environment.
 254 The test function's task is to exercise the functionality to be tested,
 255 providing it the input data from its input registers,
 256 and collecting the output in its output registers.
 257 It is the responsibility of the test environment or the framework
 258 to compare the output register values against reference values
 259 provided in the input data.
 260 .PP
 261 The input file syntax is described in full below.
 262 In overview, it is a
 263 .BR .ini -style
 264 file.
 265 Comments begin with a semicolon character
 266 .RB ` ; ',
 267 and extend to the end of the line.
 268 It is divided into
 269 .I sections
 270 by headings in square brackets:
 271 .IP
 272 .BR [ test ]
 273 .PP
 274 Each section contains a number of
 275 .I paragraphs
 276 separated by blank lines.
 277 Each paragraph consists of one or more
 278 .I assignments
 279 of the form
 280 .IP
 281 .IB reg " = " value
 282 .PP
 283 or
 284 .IP
 285 .IB reg ": " value
 286 .PP
 287 When the framework encounters a section heading,
 288 it finishes any test group currently in progress,
 289 and searches for a test definition whose name matches the
 290 .I test
 291 name in the section heading.
 292 If it finds a match,
 293 it begins a new test group with the same name.
 294 Each paragraph of assignments is used to provide
 295 input and reference output values
 296 for a single test.
 297 The
 298 .I reg
 299 name in an assignment must match the name of an active register;
 300 the corresponding
 301 .I value
 302 is stored in the named register.
 303 .PP
 304 A test environment fits in between
 305 the framework and the test function.
 306 It can establish hook functions which are called
 307 at various stages during the test group.
 308 .hP \*o
 309 The
 310 .I setup
 311 hook is called once at the start of the test group.
 312 .hP \*o
 313 The
 314 .I teardown
 315 hook is called once at the end of the test group.