Commit | Line | Data |
---|---|---|
c4ccbbf9 MW |
1 | .\" -*-nroff-*- |
2 | .\" | |
3 | .\" Manual for the test vector framework | |
4 | .\" | |
5 | .\" (c) 2024 Straylight/Edgeware | |
6 | .\" | |
7 | . | |
8 | .\"----- Licensing notice --------------------------------------------------- | |
9 | .\" | |
10 | .\" This file is part of the mLib utilities library. | |
11 | .\" | |
12 | .\" mLib is free software: you can redistribute it and/or modify it under | |
13 | .\" the terms of the GNU Library General Public License as published by | |
14 | .\" the Free Software Foundation; either version 2 of the License, or (at | |
15 | .\" your option) any later version. | |
16 | .\" | |
17 | .\" mLib is distributed in the hope that it will be useful, but WITHOUT | |
18 | .\" ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
19 | .\" FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public | |
20 | .\" License for more details. | |
21 | .\" | |
22 | .\" You should have received a copy of the GNU Library General Public | |
23 | .\" License along with mLib. If not, write to the Free Software | |
24 | .\" Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, | |
25 | .\" USA. | |
26 | . | |
27 | .\"-------------------------------------------------------------------------- | |
28 | .so ../defs.man \" @@@PRE@@@ | |
29 | . | |
30 | .\"-------------------------------------------------------------------------- | |
31 | .TH tvec 3mLib "11 March 2024" "Straylight/Edgeware" "mLib utilities library" | |
c4ccbbf9 MW |
32 | .\" @tvec_begin |
33 | .\" @tvec_end | |
34 | .\" @tvec_read | |
35 | .\" @tvec_humanoutput | |
36 | .\" @tvec_tapoutput | |
37 | .\" @tvec_dfltoutput | |
38 | . | |
39 | .\"-------------------------------------------------------------------------- | |
b1a20bee MW |
40 | .SH NAME |
41 | tvec \- test vector framework | |
42 | . | |
43 | .\"-------------------------------------------------------------------------- | |
c4ccbbf9 MW |
44 | .SH SYNOPSIS |
45 | .nf | |
46 | .B "#include <mLib/tvec.h>" | |
47 | .PP | |
48 | .ta 2n | |
49 | .B "union tvec_misc {" | |
50 | .B " const void *p;" | |
51 | .B " long i;" | |
52 | .B " unsigned long u;" | |
53 | .B " double f;" | |
54 | .B "};" | |
55 | .B "enum {" | |
56 | .B " TVMISC_PTR," | |
57 | .B " TVMISC_INT," | |
58 | .B " TVMISC_UINT," | |
59 | .B " TVMISC_FLT," | |
60 | .B " ...," | |
61 | .B " TVMISC_LIMIT," | |
62 | .B "};" | |
63 | .PP | |
64 | .ta 2n +2n | |
65 | .B "union tvec_regval {" | |
66 | .B " long i;" | |
67 | .B " unsigned long u;" | |
68 | .B " void *p;" | |
69 | .B " double f;" | |
70 | .B " struct { char *p; size_t sz; } text;" | |
71 | .B " struct { unsigned char *p; size_t sz; } bytes;" | |
72 | .B " struct {" | |
73 | .B " unsigned char *p; size_t sz;" | |
74 | .B " size_t a, m;" | |
75 | .B " size_t off;" | |
76 | .B " } buf;" | |
77 | .B " TVEC_REGSLOTS" | |
78 | .B "};" | |
79 | .B "struct tvec_reg {" | |
80 | .B " unsigned f;" | |
81 | .B " union tvec_regval v;" | |
82 | .B "};" | |
83 | .B "#define TVRF_LIVE ..." | |
84 | .PP | |
85 | .ta 2n | |
86 | .B "struct tvec_regdef {" | |
87 | .B " const char *name;" | |
88 | .B " const struct tvec_regty *ty;" | |
89 | .B " unsigned i;" | |
90 | .B " unsigned f;" | |
91 | .B " union tvec_misc arg;" | |
92 | .B "};" | |
93 | .B "#define TVRF_UNSET ..." | |
94 | .B "#define TVRF_OPT ..." | |
95 | .B "#define TVRF_ID ..." | |
96 | .B "#define TVEC_ENDREGS ..." | |
97 | .PP | |
98 | .B "struct tvec_state;" | |
99 | .PP | |
100 | .B "struct tvec_env;" | |
101 | .ta \w'\fBtypedef void tvec_testfn('u | |
102 | .BI "typedef void tvec_testfn(const struct tvec_reg *" in , | |
103 | .BI " struct tvec_reg *" out , | |
104 | .BI " void *" ctx ); | |
105 | .ta 2n | |
106 | .B "struct tvec_test {" | |
107 | .B " const char *name;" | |
108 | .B " const struct tvec_regdef *regs;" | |
109 | .B " const struct tvec_env *env;" | |
110 | .B " tvec_testfn *fn;" | |
111 | .B "};" | |
112 | .B "#define TVEC_ENDTESTS ..." | |
113 | .PP | |
114 | .ta 2n | |
115 | .B "struct tvec_config {" | |
116 | .B " const struct tvec_test *tests;" | |
117 | .B " unsigned nrout, nreg;" | |
118 | .B " size_t regsz;" | |
119 | .B "};" | |
120 | .B "struct tvec_output;" | |
121 | .PP | |
122 | .ta \w'\fBvoid tvec_begin('u | |
123 | .BI "void tvec_begin(struct tvec_state *" tv_out , | |
124 | .BI " const struct tvec_config *" config , | |
125 | .BI " struct tvec_output *" o ); | |
126 | .BI "int tvec_end(struct tvec_state *" tv ); | |
127 | .BI "int tvec_read(struct tvec_state *" tv ", const char *" infile ", FILE *" fp ); | |
128 | .PP | |
129 | .BI "extern struct tvec_output *tvec_humanoutput(FILE *" fp ); | |
5c0f2e08 | 130 | .BI "extern struct tvec_output *tvec_machineoutput(FILE *" fp ); |
c4ccbbf9 MW |
131 | .BI "extern struct tvec_output *tvec_tapoutput(FILE *" fp ); |
132 | .BI "extern struct tvec_output *tvec_dfltoutput(FILE *" fp ); | |
133 | .fi | |
134 | . | |
135 | .\"-------------------------------------------------------------------------- | |
136 | .SH DESCRIPTION | |
137 | . | |
138 | The | |
139 | .B <mLib/tvec.h> | |
140 | header file provides definitions and declarations | |
141 | for the core of mLib's | |
142 | .IR "test vector framework" . | |
143 | .PP | |
144 | The test vector framework is rather large and complicated, | |
145 | so the documentation for it is split into multiple manual pages. | |
146 | This one provides a conceptual overview | |
147 | and describes the essentials for using it to build simple tests. | |
148 | . | |
149 | .SS Conceptual overview | |
150 | A | |
151 | .I "test session" | |
152 | runs a collection of tests | |
153 | and reports on the outcome. | |
154 | .PP | |
155 | A | |
156 | .I test | |
157 | involves exercising some functionality | |
158 | and checking that it behaves properly. | |
159 | A test can have four | |
160 | .IR outcomes . | |
161 | It can | |
162 | .IR pass : | |
163 | the functionality behaved properly. | |
164 | It can | |
165 | .IR fail : | |
166 | the functionality did not behave properly. | |
167 | It can experience an | |
168 | .IR "expected failure" : | |
169 | the functionality behaved as expected, | |
170 | but the expected behaviour is known to be incorrect. | |
171 | Or it can be | |
172 | .IR skipped : | |
173 | for some reason, the test couldn't be performed. | |
174 | .PP | |
175 | Tests are gathered together into | |
176 | .IR "test groups" . | |
177 | Each test group has a name. | |
178 | Like a individual tests, test groups also have outcomes: | |
179 | they can pass, fail, or be skipped. | |
180 | A test group cannot experience expected failure. | |
181 | .PP | |
182 | A session may also encounter | |
183 | .IR errors , | |
184 | e.g., as a result of malformed input | |
185 | or failures reported by system facilities. | |
186 | .PP | |
187 | A test session can either | |
188 | be driven from data provided by an input file, | |
189 | or it can be driven by the program alone. | |
190 | The latter case is called | |
191 | .I "ad-hoc testing", | |
192 | and is described in | |
193 | .BR tvec-adhoc (3). | |
194 | This manual page describes file-driven testing. | |
195 | .PP | |
196 | When it begins a session for file-directed testing, | |
197 | the program provides a table of | |
198 | .IR "test definitions" . | |
199 | A test definition has a | |
200 | .IR name , | |
201 | and also specifies a | |
202 | .IR "test function" , | |
203 | a | |
204 | .IR "test environment" , | |
205 | and a table of | |
206 | .IR "register definitions" . | |
207 | Test environments are explained further below. | |
208 | .PP | |
209 | A | |
210 | .I register | |
211 | is a place which can store a single item of test data; | |
212 | registers are the means | |
213 | by which input test data is provided to a test function, | |
214 | and by which a test function returns its results. | |
215 | A test definition's register definitions | |
216 | establish a collection of | |
217 | .I active | |
218 | registers. | |
219 | Each active register has a | |
220 | .IR name , | |
221 | an | |
222 | .IR index , | |
223 | and a | |
224 | .IR type , | |
225 | which are established by its register definition. | |
226 | The register's name is used to refer to the register in the test data file, | |
227 | and its index is used to refer to it | |
228 | in the test function and test environments. | |
229 | The register's type describes the acceptable values for the register, | |
230 | and how they are to be compared, | |
231 | read from the input file, | |
232 | and dumped in diagnostic output. | |
233 | New register types can be defined fairly easily: see | |
234 | .BR tvec_tyimpl (3) | |
235 | for the details. | |
236 | A register definition may describe an | |
237 | .I input | |
238 | register or an | |
239 | .I output | |
240 | register: | |
241 | input registers provide input data to the test function, while | |
242 | output registers collect output data from the test function. | |
243 | The data file provides values for both input and output registers: | |
244 | the values for the input registers are passed to the test function; | |
245 | the values for the output registers are | |
246 | .I "reference outputs" | |
247 | against which the test function's outputs are to be checked. | |
248 | .PP | |
249 | The test function is called with two vectors of registers, | |
b1a20bee MW |
250 | one containing input values for the test function to read |
251 | and also reference output values, | |
c4ccbbf9 MW |
252 | and another for output values that the test function should write; |
253 | and a | |
254 | .I context | |
255 | provided by the test environment. | |
256 | The test function's task is to exercise the functionality to be tested, | |
257 | providing it the input data from its input registers, | |
258 | and collecting the output in its output registers. | |
259 | It is the responsibility of the test environment or the framework | |
260 | to compare the output register values against reference values | |
261 | provided in the input data. | |
262 | .PP | |
263 | The input file syntax is described in full below. | |
264 | In overview, it is a | |
265 | .BR .ini -style | |
266 | file. | |
267 | Comments begin with a semicolon character | |
268 | .RB ` ; ', | |
269 | and extend to the end of the line. | |
270 | It is divided into | |
271 | .I sections | |
272 | by headings in square brackets: | |
273 | .IP | |
274 | .BR [ test ] | |
275 | .PP | |
276 | Each section contains a number of | |
277 | .I paragraphs | |
278 | separated by blank lines. | |
279 | Each paragraph consists of one or more | |
280 | .I assignments | |
281 | of the form | |
282 | .IP | |
283 | .IB reg " = " value | |
284 | .PP | |
285 | or | |
286 | .IP | |
287 | .IB reg ": " value | |
288 | .PP | |
289 | When the framework encounters a section heading, | |
290 | it finishes any test group currently in progress, | |
291 | and searches for a test definition whose name matches the | |
292 | .I test | |
293 | name in the section heading. | |
294 | If it finds a match, | |
295 | it begins a new test group with the same name. | |
296 | Each paragraph of assignments is used to provide | |
297 | input and reference output values | |
298 | for a single test. | |
299 | The | |
300 | .I reg | |
b1a20bee MW |
301 | name in an assignment must match the name |
302 | of an active register or a | |
303 | .I "special variable" ; | |
c4ccbbf9 MW |
304 | the corresponding |
305 | .I value | |
b1a20bee MW |
306 | is stored in the named register or variable. |
307 | .PP | |
308 | A register which has been assigned a value is said to be | |
309 | .IR live ; | |
310 | otherwise, it is | |
311 | .IR dead . | |
312 | By default, every active register must be live for a test to proceed; | |
313 | but a register definition can mark its register as | |
314 | .I optional | |
315 | or | |
316 | .IR may-be-unset . | |
317 | An optional register need not be assigned a value explicitly; | |
318 | instead, the register is left dead. | |
319 | A may-be-unset register must be mentioned, | |
320 | but a distinctive syntax | |
321 | .IP | |
322 | .IB reg " *" | |
323 | .PP | |
324 | (with no colon or equals sign) | |
325 | says that the register should be left dead. | |
326 | Optional registers are suitable for cases where | |
327 | there is an obvious default value | |
328 | which would otherwise be mentioned frequently in input files. | |
329 | May-be-unset registers are mostly useful as outputs, | |
330 | where the output is not always set, | |
331 | e.g., in error cases, | |
332 | but where omitting a value in the usual case is likely a mistake. | |
c4ccbbf9 | 333 | .PP |
6e683a79 MW |
334 | A test environment fits in between |
335 | the framework and the test function. | |
336 | It can establish hook functions which are called | |
b1a20bee MW |
337 | at various points throughout a test group |
338 | (at the start and and, and before and after each test). | |
339 | It can define special variables | |
340 | which can be set from the input file using assignments. | |
341 | And, finally, it can take on the responsibility | |
342 | of running the test function. | |
343 | The registers will have been set up already, | |
344 | based on the assignments in the input file, | |
345 | but the environment can modify them. | |
346 | It must also check the test function's output | |
347 | against the reference values, | |
348 | though there are functions provided for doing this. | |
349 | The environment can choose to run the test function once, | |
350 | multiple times, or, indeed, not at all. | |
351 | When it calls the test function, it can provide a context pointer, | |
352 | with whatever additional information might be useful: | |
353 | this usually involves coordination | |
354 | between the environment and the test function. | |
355 | It is the test environment's responsibility | |
356 | to check the outputs returned by the test function | |
357 | and to report on mismatches, | |
358 | but there are functions provided by the framework | |
359 | to do the heavy lifting. | |
360 | .PP | |
361 | The following are examples of what test environments can do. | |
6e683a79 | 362 | .hP \*o |
b1a20bee MW |
363 | It can fill in default values for optional dead registers. |
364 | For example, if a function returnns error codes, | |
365 | then you can save reptition in the input file | |
366 | by marking the error-code output register optional | |
367 | and letting it default to the `success' value in a test environment. | |
368 | .hP \*o | |
369 | It can run the test function mulitple times. | |
370 | For example, a test of functions like | |
371 | .BR strcmp (3) | |
372 | might run the test twice, | |
373 | first with its operands as supplied by the input file, | |
374 | and then again with the operands swapped | |
375 | and the opposite expected result. | |
376 | A test for bignum addition could verify commutativity | |
377 | by checking both that | |
378 | .IR x "\ + \ " y "\ =\ " z | |
379 | and that | |
380 | .IR y "\ + \ " x "\ =\ " z \fR. | |
381 | Similarly, a subtraction test could check both that | |
382 | .IR x "\ \- \ " y "\ =\ " z | |
383 | and that | |
384 | .IR y "\ \- \ " x "\ =\ \-" z \fR. | |
6e683a79 MW |
385 | .hP \*o |
386 | The | |
b1a20bee MW |
387 | .BR tvec-remote (3), |
388 | .BR tvec-timeout (3), | |
389 | and | |
390 | .BR tvec-bench (3) | |
391 | extensions all slot in as test environments. | |
392 | Sadly, | |
393 | .BR tvec-bench (3) | |
394 | requires support from the output protocol | |
395 | in order to format benchmark results properly; | |
396 | apart from that, | |
397 | these three facilities are pure extensions |