@@@ misc wip

[mLib] / test / tvec.3.in

.\" -*-nroff-*-
.\"
.\" Manual for the test vector framework
.\"
.\" (c) 2024 Straylight/Edgeware
.\"
.
.\"----- Licensing notice ---------------------------------------------------
.\"
.\" This file is part of the mLib utilities library.
.\"
.\" mLib is free software: you can redistribute it and/or modify it under
.\" the terms of the GNU Library General Public License as published by
.\" the Free Software Foundation; either version 2 of the License, or (at
.\" your option) any later version.
.\"
.\" mLib is distributed in the hope that it will be useful, but WITHOUT
.\" ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
.\" FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
.\" License for more details.
.\"
.\" You should have received a copy of the GNU Library General Public
.\" License along with mLib.  If not, write to the Free Software
.\" Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
.\" USA.
.
.\"--------------------------------------------------------------------------
.so ../defs.man \" @@@PRE@@@
.
.\"--------------------------------------------------------------------------
.TH tvec 3mLib "11 March 2024" "Straylight/Edgeware" "mLib utilities library"
.
.\"--------------------------------------------------------------------------
.SH NAME
tvec \- test vector framework
.\" @tvec_begin
.\" @tvec_end
.\" @tvec_read
.\" @tvec_humanoutput
.\" @tvec_tapoutput
.\" @tvec_dfltoutput
.
.\"--------------------------------------------------------------------------
.SH SYNOPSIS
.nf
.B "#include <mLib/tvec.h>"
.PP
.ta 2n
.B "union tvec_misc {"
.B "	const void *p;"
.B "	long i;"
.B "	unsigned long u;"
.B "	double f;"
.B "};"
.B "enum {"
.B "	TVMISC_PTR,"
.B "	TVMISC_INT,"
.B "	TVMISC_UINT,"
.B "	TVMISC_FLT,"
.B "	...,"
.B "	TVMISC_LIMIT,"
.B "};"
.PP
.ta 2n +2n
.B "union tvec_regval {"
.B "	long i;"
.B "	unsigned long u;"
.B "	void *p;"
.B "	double f;"
.B "	struct { char *p; size_t sz; } text;"
.B "	struct { unsigned char *p; size_t sz; } bytes;"
.B "	struct {"
.B "		unsigned char *p; size_t sz;"
.B "		size_t a, m;"
.B "		size_t off;"
.B "	} buf;"
.B "	TVEC_REGSLOTS"
.B "};"
.B "struct tvec_reg {"
.B "	unsigned f;"
.B "	union tvec_regval v;"
.B "};"
.B "#define TVRF_LIVE ..."
.PP
.ta 2n
.B "struct tvec_regdef {"
.B "	const char *name;"
.B "	const struct tvec_regty *ty;"
.B "	unsigned i;"
.B "	unsigned f;"
.B "	union tvec_misc arg;"
.B "};"
.B "#define TVRF_UNSET ..."
.B "#define TVRF_OPT ..."
.B "#define TVRF_ID ..."
.B "#define TVEC_ENDREGS ..."
.PP
.B "struct tvec_state;"
.PP
.B "struct tvec_env;"
.ta \w'\fBtypedef void tvec_testfn('u
.BI "typedef void tvec_testfn(const struct tvec_reg *" in ,
.BI "	struct tvec_reg *" out ,
.BI "	void *" ctx );
.ta 2n
.B "struct tvec_test {"
.B "	const char *name;"
.B "	const struct tvec_regdef *regs;"
.B "	const struct tvec_env *env;"
.B "	tvec_testfn *fn;"
.B "};"
.B "#define TVEC_ENDTESTS ..."
.PP
.ta 2n
.B "struct tvec_config {"
.B "	const struct tvec_test *tests;"
.B "	unsigned nrout, nreg;"
.B "	size_t regsz;"
.B "};"
.B "struct tvec_output;"
.PP
.ta \w'\fBvoid tvec_begin('u
.BI "void tvec_begin(struct tvec_state *" tv_out ,
.BI "	const struct tvec_config *" config ,
.BI "	struct tvec_output *" o );
.BI "int tvec_end(struct tvec_state *" tv );
.BI "int tvec_read(struct tvec_state *" tv ", const char *" infile ", FILE *" fp );
.PP
.BI "extern struct tvec_output *tvec_humanoutput(FILE *" fp );
.BI "extern struct tvec_output *tvec_machineoutput(FILE *" fp );
.BI "extern struct tvec_output *tvec_tapoutput(FILE *" fp );
.BI "extern struct tvec_output *tvec_dfltoutput(FILE *" fp );
.fi
.
.\"--------------------------------------------------------------------------
.SH DESCRIPTION
.
The
.B <mLib/tvec.h>
header file provides definitions and declarations
for the core of mLib's
.IR "test vector framework" .
.PP
The test vector framework is rather large and complicated,
so the documentation for it is split into multiple manual pages.
This one provides a conceptual overview
and describes the essentials for using it to build simple tests.
.
.SS Conceptual overview
A
.I "test session"
runs a collection of tests
and reports on the outcome.
.PP
A
.I test
involves exercising some functionality
and checking that it behaves properly.
A test can have four
.IR outcomes .
It can
.IR pass :
the functionality behaved properly.
It can
.IR fail :
the functionality did not behave properly.
It can experience an
.IR "expected failure" :
the functionality behaved as expected,
but the expected behaviour is known to be incorrect.
Or it can be
.IR skipped :
for some reason, the test couldn't be performed.
.PP
Tests are gathered together into
.IR "test groups" .
Each test group has a name.
Like a individual tests, test groups also have outcomes:
they can pass, fail, or be skipped.
A test group cannot experience expected failure.
.PP
A session may also encounter
.IR errors ,
e.g., as a result of malformed input
or failures reported by system facilities.
.PP
A test session can either
be driven from data provided by an input file,
or it can be driven by the program alone.
The latter case is called
.I "ad-hoc testing",
and is described in
.BR tvec-adhoc (3).
This manual page describes file-driven testing.
.PP
When it begins a session for file-directed testing,
the program provides a table of
.IR "test definitions" .
A test definition has a
.IR name ,
and also specifies a
.IR "test function" ,
a
.IR "test environment" ,
and a table of
.IR "register definitions" .
Test environments are explained further below.
.PP
A
.I register
is a place which can store a single item of test data;
registers are the means
by which input test data is provided to a test function,
and by which a test function returns its results.
A test definition's register definitions
establish a collection of
.I active
registers.
Each active register has a
.IR name ,
an
.IR index ,
and a
.IR type ,
which are established by its register definition.
The register's name is used to refer to the register in the test data file,
and its index is used to refer to it
in the test function and test environments.
The register's type describes the acceptable values for the register,
and how they are to be compared,
read from the input file,
and dumped in diagnostic output.
New register types can be defined fairly easily: see
.BR tvec_tyimpl (3)
for the details.
A register definition may describe an
.I input
register or an
.I output
register:
input registers provide input data to the test function, while
output registers collect output data from the test function.
The data file provides values for both input and output registers:
the values for the input registers are passed to the test function;
the values for the output registers are
.I "reference outputs"
against which the test function's outputs are to be checked.
.PP
The test function is called with two vectors of registers,
one containing input values for the test function to read,
and another for output values that the test function should write;
and a
.I context
provided by the test environment.
The test function's task is to exercise the functionality to be tested,
providing it the input data from its input registers,
and collecting the output in its output registers.
It is the responsibility of the test environment or the framework
to compare the output register values against reference values
provided in the input data.
.PP
The input file syntax is described in full below.
In overview, it is a
.BR .ini -style
file.
Comments begin with a semicolon character
.RB ` ; ',
and extend to the end of the line.
It is divided into
.I sections
by headings in square brackets:
.IP
.BR [ test ]
.PP
Each section contains a number of
.I paragraphs
separated by blank lines.
Each paragraph consists of one or more
.I assignments
of the form
.IP
.IB reg " = " value
.PP
or
.IP
.IB reg ": " value
.PP
When the framework encounters a section heading,
it finishes any test group currently in progress,
and searches for a test definition whose name matches the
.I test
name in the section heading.
If it finds a match,
it begins a new test group with the same name.
Each paragraph of assignments is used to provide
input and reference output values
for a single test.
The
.I reg
name in an assignment must match the name of an active register;
the corresponding
.I value
is stored in the named register.
.PP
A test environment fits in between
the framework and the test function.
It can establish hook functions which are called
at various stages during the test group.
.hP \*o
The
.I setup
hook is called once at the start of the test group.
.hP \*o
The
.I teardown
hook is called once at the end of the test group.