Commit | Line | Data |
---|---|---|
c4ccbbf9 MW |
1 | .\" -*-nroff-*- |
2 | .\" | |
3 | .\" Manual for the test vector framework | |
4 | .\" | |
5 | .\" (c) 2024 Straylight/Edgeware | |
6 | .\" | |
7 | . | |
8 | .\"----- Licensing notice --------------------------------------------------- | |
9 | .\" | |
10 | .\" This file is part of the mLib utilities library. | |
11 | .\" | |
12 | .\" mLib is free software: you can redistribute it and/or modify it under | |
13 | .\" the terms of the GNU Library General Public License as published by | |
14 | .\" the Free Software Foundation; either version 2 of the License, or (at | |
15 | .\" your option) any later version. | |
16 | .\" | |
17 | .\" mLib is distributed in the hope that it will be useful, but WITHOUT | |
18 | .\" ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
19 | .\" FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public | |
20 | .\" License for more details. | |
21 | .\" | |
22 | .\" You should have received a copy of the GNU Library General Public | |
23 | .\" License along with mLib. If not, write to the Free Software | |
24 | .\" Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, | |
25 | .\" USA. | |
26 | . | |
27 | .\"-------------------------------------------------------------------------- | |
28 | .so ../defs.man \" @@@PRE@@@ | |
29 | . | |
30 | .\"-------------------------------------------------------------------------- | |
31 | .TH tvec 3mLib "11 March 2024" "Straylight/Edgeware" "mLib utilities library" | |
32 | . | |
33 | .\"-------------------------------------------------------------------------- | |
34 | .SH NAME | |
35 | tvec \- test vector framework | |
36 | .\" @tvec_begin | |
37 | .\" @tvec_end | |
38 | .\" @tvec_read | |
39 | .\" @tvec_humanoutput | |
40 | .\" @tvec_tapoutput | |
41 | .\" @tvec_dfltoutput | |
42 | . | |
43 | .\"-------------------------------------------------------------------------- | |
44 | .SH SYNOPSIS | |
45 | .nf | |
46 | .B "#include <mLib/tvec.h>" | |
47 | .PP | |
48 | .ta 2n | |
49 | .B "union tvec_misc {" | |
50 | .B " const void *p;" | |
51 | .B " long i;" | |
52 | .B " unsigned long u;" | |
53 | .B " double f;" | |
54 | .B "};" | |
55 | .B "enum {" | |
56 | .B " TVMISC_PTR," | |
57 | .B " TVMISC_INT," | |
58 | .B " TVMISC_UINT," | |
59 | .B " TVMISC_FLT," | |
60 | .B " ...," | |
61 | .B " TVMISC_LIMIT," | |
62 | .B "};" | |
63 | .PP | |
64 | .ta 2n +2n | |
65 | .B "union tvec_regval {" | |
66 | .B " long i;" | |
67 | .B " unsigned long u;" | |
68 | .B " void *p;" | |
69 | .B " double f;" | |
70 | .B " struct { char *p; size_t sz; } text;" | |
71 | .B " struct { unsigned char *p; size_t sz; } bytes;" | |
72 | .B " struct {" | |
73 | .B " unsigned char *p; size_t sz;" | |
74 | .B " size_t a, m;" | |
75 | .B " size_t off;" | |
76 | .B " } buf;" | |
77 | .B " TVEC_REGSLOTS" | |
78 | .B "};" | |
79 | .B "struct tvec_reg {" | |
80 | .B " unsigned f;" | |
81 | .B " union tvec_regval v;" | |
82 | .B "};" | |
83 | .B "#define TVRF_LIVE ..." | |
84 | .PP | |
85 | .ta 2n | |
86 | .B "struct tvec_regdef {" | |
87 | .B " const char *name;" | |
88 | .B " const struct tvec_regty *ty;" | |
89 | .B " unsigned i;" | |
90 | .B " unsigned f;" | |
91 | .B " union tvec_misc arg;" | |
92 | .B "};" | |
93 | .B "#define TVRF_UNSET ..." | |
94 | .B "#define TVRF_OPT ..." | |
95 | .B "#define TVRF_ID ..." | |
96 | .B "#define TVEC_ENDREGS ..." | |
97 | .PP | |
98 | .B "struct tvec_state;" | |
99 | .PP | |
100 | .B "struct tvec_env;" | |
101 | .ta \w'\fBtypedef void tvec_testfn('u | |
102 | .BI "typedef void tvec_testfn(const struct tvec_reg *" in , | |
103 | .BI " struct tvec_reg *" out , | |
104 | .BI " void *" ctx ); | |
105 | .ta 2n | |
106 | .B "struct tvec_test {" | |
107 | .B " const char *name;" | |
108 | .B " const struct tvec_regdef *regs;" | |
109 | .B " const struct tvec_env *env;" | |
110 | .B " tvec_testfn *fn;" | |
111 | .B "};" | |
112 | .B "#define TVEC_ENDTESTS ..." | |
113 | .PP | |
114 | .ta 2n | |
115 | .B "struct tvec_config {" | |
116 | .B " const struct tvec_test *tests;" | |
117 | .B " unsigned nrout, nreg;" | |
118 | .B " size_t regsz;" | |
119 | .B "};" | |
120 | .B "struct tvec_output;" | |
121 | .PP | |
122 | .ta \w'\fBvoid tvec_begin('u | |
123 | .BI "void tvec_begin(struct tvec_state *" tv_out , | |
124 | .BI " const struct tvec_config *" config , | |
125 | .BI " struct tvec_output *" o ); | |
126 | .BI "int tvec_end(struct tvec_state *" tv ); | |
127 | .BI "int tvec_read(struct tvec_state *" tv ", const char *" infile ", FILE *" fp ); | |
128 | .PP | |
129 | .BI "extern struct tvec_output *tvec_humanoutput(FILE *" fp ); | |
130 | .BI "extern struct tvec_output *tvec_tapoutput(FILE *" fp ); | |
131 | .BI "extern struct tvec_output *tvec_dfltoutput(FILE *" fp ); | |
132 | .fi | |
133 | . | |
134 | .\"-------------------------------------------------------------------------- | |
135 | .SH DESCRIPTION | |
136 | . | |
137 | The | |
138 | .B <mLib/tvec.h> | |
139 | header file provides definitions and declarations | |
140 | for the core of mLib's | |
141 | .IR "test vector framework" . | |
142 | .PP | |
143 | The test vector framework is rather large and complicated, | |
144 | so the documentation for it is split into multiple manual pages. | |
145 | This one provides a conceptual overview | |
146 | and describes the essentials for using it to build simple tests. | |
147 | . | |
148 | .SS Conceptual overview | |
149 | A | |
150 | .I "test session" | |
151 | runs a collection of tests | |
152 | and reports on the outcome. | |
153 | .PP | |
154 | A | |
155 | .I test | |
156 | involves exercising some functionality | |
157 | and checking that it behaves properly. | |
158 | A test can have four | |
159 | .IR outcomes . | |
160 | It can | |
161 | .IR pass : | |
162 | the functionality behaved properly. | |
163 | It can | |
164 | .IR fail : | |
165 | the functionality did not behave properly. | |
166 | It can experience an | |
167 | .IR "expected failure" : | |
168 | the functionality behaved as expected, | |
169 | but the expected behaviour is known to be incorrect. | |
170 | Or it can be | |
171 | .IR skipped : | |
172 | for some reason, the test couldn't be performed. | |
173 | .PP | |
174 | Tests are gathered together into | |
175 | .IR "test groups" . | |
176 | Each test group has a name. | |
177 | Like a individual tests, test groups also have outcomes: | |
178 | they can pass, fail, or be skipped. | |
179 | A test group cannot experience expected failure. | |
180 | .PP | |
181 | A session may also encounter | |
182 | .IR errors , | |
183 | e.g., as a result of malformed input | |
184 | or failures reported by system facilities. | |
185 | .PP | |
186 | A test session can either | |
187 | be driven from data provided by an input file, | |
188 | or it can be driven by the program alone. | |
189 | The latter case is called | |
190 | .I "ad-hoc testing", | |
191 | and is described in | |
192 | .BR tvec-adhoc (3). | |
193 | This manual page describes file-driven testing. | |
194 | .PP | |
195 | When it begins a session for file-directed testing, | |
196 | the program provides a table of | |
197 | .IR "test definitions" . | |
198 | A test definition has a | |
199 | .IR name , | |
200 | and also specifies a | |
201 | .IR "test function" , | |
202 | a | |
203 | .IR "test environment" , | |
204 | and a table of | |
205 | .IR "register definitions" . | |
206 | Test environments are explained further below. | |
207 | .PP | |
208 | A | |
209 | .I register | |
210 | is a place which can store a single item of test data; | |
211 | registers are the means | |
212 | by which input test data is provided to a test function, | |
213 | and by which a test function returns its results. | |
214 | A test definition's register definitions | |
215 | establish a collection of | |
216 | .I active | |
217 | registers. | |
218 | Each active register has a | |
219 | .IR name , | |
220 | an | |
221 | .IR index , | |
222 | and a | |
223 | .IR type , | |
224 | which are established by its register definition. | |
225 | The register's name is used to refer to the register in the test data file, | |
226 | and its index is used to refer to it | |
227 | in the test function and test environments. | |
228 | The register's type describes the acceptable values for the register, | |
229 | and how they are to be compared, | |
230 | read from the input file, | |
231 | and dumped in diagnostic output. | |
232 | New register types can be defined fairly easily: see | |
233 | .BR tvec_tyimpl (3) | |
234 | for the details. | |
235 | A register definition may describe an | |
236 | .I input | |
237 | register or an | |
238 | .I output | |
239 | register: | |
240 | input registers provide input data to the test function, while | |
241 | output registers collect output data from the test function. | |
242 | The data file provides values for both input and output registers: | |
243 | the values for the input registers are passed to the test function; | |
244 | the values for the output registers are | |
245 | .I "reference outputs" | |
246 | against which the test function's outputs are to be checked. | |
247 | .PP | |
248 | The test function is called with two vectors of registers, | |
249 | one containing input values for the test function to read, | |
250 | and another for output values that the test function should write; | |
251 | and a | |
252 | .I context | |
253 | provided by the test environment. | |
254 | The test function's task is to exercise the functionality to be tested, | |
255 | providing it the input data from its input registers, | |
256 | and collecting the output in its output registers. | |
257 | It is the responsibility of the test environment or the framework | |
258 | to compare the output register values against reference values | |
259 | provided in the input data. | |
260 | .PP | |
261 | The input file syntax is described in full below. | |
262 | In overview, it is a | |
263 | .BR .ini -style | |
264 | file. | |
265 | Comments begin with a semicolon character | |
266 | .RB ` ; ', | |
267 | and extend to the end of the line. | |
268 | It is divided into | |
269 | .I sections | |
270 | by headings in square brackets: | |
271 | .IP | |
272 | .BR [ test ] | |
273 | .PP | |
274 | Each section contains a number of | |
275 | .I paragraphs | |
276 | separated by blank lines. | |
277 | Each paragraph consists of one or more | |
278 | .I assignments | |
279 | of the form | |
280 | .IP | |
281 | .IB reg " = " value | |
282 | .PP | |
283 | or | |
284 | .IP | |
285 | .IB reg ": " value | |
286 | .PP | |
287 | When the framework encounters a section heading, | |
288 | it finishes any test group currently in progress, | |
289 | and searches for a test definition whose name matches the | |
290 | .I test | |
291 | name in the section heading. | |
292 | If it finds a match, | |
293 | it begins a new test group with the same name. | |
294 | Each paragraph of assignments is used to provide | |
295 | input and reference output values | |
296 | for a single test. | |
297 | The | |
298 | .I reg | |
299 | name in an assignment must match the name of an active register; | |
300 | the corresponding | |
301 | .I value | |
302 | is stored in the named register. | |
303 | .PP |