3 * Benchmarking in the test-vector framework
5 * (c) 2023 Straylight/Edgeware
8 /*----- Licensing notice --------------------------------------------------*
10 * This file is part of the mLib utilities library.
12 * mLib is free software: you can redistribute it and/or modify it under
13 * the terms of the GNU Library General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or (at
15 * your option) any later version.
17 * mLib is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
20 * License for more details.
22 * You should have received a copy of the GNU Library General Public
23 * License along with mLib. If not, write to the Free Software
24 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
28 /*----- Header files ------------------------------------------------------*/
33 #include "tvec-bench.h"
34 #include "tvec-output.h"
35 #include "tvec-types.h"
37 /*----- Data structures ---------------------------------------------------*/
40 struct tvec_state
*tv
; /* test vector state */
41 const struct tvec_env
*env
; /* subordinate environment */
42 void *ctx
; /* subordinate env's context */
43 unsigned long *n
; /* iteration count address */
44 const struct tvec_reg
*in
; struct tvec_reg
*out
; /* register vectors */
45 tvec_testfn
*fn
; /* test function to run */
48 /*----- Global variables --------------------------------------------------*/
50 struct bench_state
*tvec_benchstate
; /* common benchmarking state */
52 /*----- Output utilities --------------------------------------------------*/
54 /* --- @tvec_benchreport@ --- *
56 * Arguments: @const struct gprintf_ops *gops@ = print operations
57 * @void *go@ = print destination
58 * @unsigned unit@ = the unit being measured (@TVBU_...@)
59 * @unsigned style@ = output style (@TVSF_...@)
60 * @const struct bench_timing *t@ = the benchmark result
64 * Use: Formats a report about the benchmark performance. This
65 * function is intended to be called on by an output
69 void tvec_benchreport(const struct gprintf_ops
*gops
, void *go
,
70 unsigned unit
, unsigned style
,
71 const struct bench_timing
*t
)
74 if (style
&TVSF_RAW
) gprintf(gops
, go
, "FAILED");
75 else gprintf(gops
, go
, "benchmark FAILED");
76 } else if (!(style
&TVSF_RAW
))
77 bench_report(gops
, go
, unit
, t
);
80 case BTU_OP
: gprintf(gops
, go
, "ops=%.0f", t
->n
); break;
81 case BTU_BYTE
: gprintf(gops
, go
, "bytes=%.0f", t
->n
); break;
84 gprintf(gops
, go
, " sec=%.6g", t
->t
);
85 if (t
->f
&BTF_CYOK
) gprintf(gops
, go
, " cy=%.0f", t
->cy
);
89 /*----- Default output implementation -------------------------------------*/
91 /* --- @fallback_bbench@ --- *
93 * Arguments: @struct tvec_output *o@ = output sink, secretly a
94 * @struct fallback_output@
95 * @const char *desc@ = adhoc test description
96 * @unsigned unit@ = measurement unit (@TVBU_...@)
100 * Use: Report that a benchmark has started.
102 * The fallback implementation does nothing here. All of the
103 * reporting happens in @fallback_ebench@.
106 static void fallback_bbench(struct tvec_output
*o
,
107 const char *desc
, unsigned unit
)
110 /* --- @fallback_ebench@ --- *
112 * Arguments: @struct tvec_output *o@ = output sink, secretly a
113 * @struct fallback_output@
114 * @const char *desc@ = adhoc test description
115 * @unsigned unit@ = measurement unit (@BTU_...@)
116 * @const struct bench_timing *t@ = measurement
120 * Use: Report a benchmark's results
122 * The fallback implementation just delegates to the default
123 * benchmark reporting to produce a line written through the
124 * standard @report@ output operation.
127 static void fallback_ebench(struct tvec_output
*o
,
128 const char *desc
, unsigned unit
,
129 const struct bench_timing
*t
)
131 struct tvec_fallbackoutput
*fo
= (struct tvec_fallbackoutput
*)o
;
132 struct tvec_state
*tv
= fo
->tv
;
133 const struct tvec_regdef
*rd
;
138 /* Build the identification string. */
139 dstr_putf(&d
, "%s ", tv
->test
->name
);
143 for (rd
= tv
->test
->regs
; rd
->name
; rd
++)
145 if (f
&f_any
) DPUTS(&d
, ", ");
147 dstr_putf(&d
, "%s = ", rd
->name
);
148 rd
->ty
->dump(&TVEC_REG(tv
, in
, rd
->i
)->v
, rd
, TVSF_COMPACT
,
153 /* Report the benchmark results. */
154 tvec_benchreport(&dstr_printops
, &d
, unit
, 0, t
);
157 /* Write the result. */
158 tvec_info(tv
, "%s", d
.buf
);
164 const struct tvec_benchoutops tvec_benchoutputfallback
=
165 { fallback_bbench
, fallback_ebench
};
167 /*----- Benchmark environment scaffolding ---------------------------------*/
169 /* --- @tvec_benchprep@ --- *
171 * Arguments: @struct tvec_state *tv@ = test vector state
172 * @struct bench_state **bst_inout@ = benchmark state (updated)
173 * @unsigned f@ = calibration flags
175 * Returns: Zero on success, %$-1$% on failure.
177 * Use: If @*bst_inout@ is null then allocate and initialize a fresh
178 * benchmark state with a default timer, and @*bst_inout@ is
179 * updated to point to the fresh state. The storage for the
180 * state was allocated using the test vector state's arena.
182 * If the benchmark state hasn't been calibrated, then this is
183 * done, passing @f@ to @bench_calibrate@.
185 * On failure, the test group is skipped, reporting a suitable
186 * message, and %$-1$% is returned. If a fresh benchmark state
187 * was allocated, but calibration failed, the state is
188 * %%\emph{not}%% released.
191 int tvec_benchprep(struct tvec_state
*tv
,
192 struct bench_state
**bst_inout
, unsigned f
)
195 struct bench_timer
*bt
;
196 struct bench_state
*b
;
199 /* Set up the benchmarking state if it hasn't been done before. */
203 bt
= bench_createtimer(0);
205 { tvec_skipgroup(tv
, "failed to create timer"); goto timer_failed
; }
206 X_NEW(b
, tv
->a
); bench_init(b
, bt
); *bst_inout
= b
;
209 /* If the timer isn't calibrated yet then do that now. */
210 if (!(b
->f
&BTF_CLB
)) {
211 b
->tm
->ops
->describe(b
->tm
, &d
);
212 tvec_notice(tv
, "calibrating timer `%s'...", d
.buf
);
213 if (bench_calibrate(b
, f
))
214 { tvec_skipgroup(tv
, "failed to calibrate timer"); goto timer_failed
; }
215 } else if (!(b
->f
&BTF_ANY
))
216 { tvec_skipgroup(tv
, "timer broken"); goto timer_failed
; }
221 /* Give a `helpful' hint if the timer didn't work. */
222 if (!getenv("MLIB_BENCH_DEBUG"))
223 tvec_notice(tv
, "set `MLIB_BENCH_DEBUG=t' in the environment "
233 /* --- @tvec_benchsetup@ --- *
235 * Arguments: @struct tvec_state *tv@ = test vector state
236 * @const struct tvec_env *env@ = environment description
237 * @void *pctx@ = parent context (ignored)
238 * @void *ctx@ = context pointer to initialize
242 * Use: Initialize a benchmarking environment context.
244 * The environment description must really be a @struct
245 * tvec_benchenv@. If the @bst@ slot is null, then a temporary
246 * benchmark state is allocated for the current test group and
247 * released at the end. Otherwise, it must be the address of a
248 * pointer to a benchmark state: if the pointer is null, then a
249 * fresh state is allocated and initialized and the pointer is
250 * updated; otherwise, the pointer is assumed to refer to an
251 * existing valid benchmark state.
254 void tvec_benchsetup(struct tvec_state
*tv
, const struct tvec_env
*env
,
255 void *pctx
, void *ctx
)
257 struct tvec_benchctx
*bc
= ctx
;
258 const struct tvec_benchenv
*be
= (const struct tvec_benchenv
*)env
;
259 const struct tvec_env
*subenv
= be
->env
;
261 bc
->be
= be
; bc
->bst
= 0; bc
->subctx
= 0;
262 if (!tvec_benchprep(tv
, be
->bst
, BTF_INDIRECT
))
263 { bc
->bst
= *be
->bst
; bc
->dflt_target
= bc
->bst
->target_s
; }
264 if (subenv
&& subenv
->ctxsz
) bc
->subctx
= x_alloc(tv
->a
, subenv
->ctxsz
);
265 if (subenv
&& subenv
->setup
) subenv
->setup(tv
, subenv
, bc
, bc
->subctx
);
268 /* --- @tvec_benchfindvar@, @setvar@ --- *
270 * Arguments: @struct tvec_state *tv@ = test vector state
271 * @const char *var@ = variable name to set
272 * @const union tvec_regval *rv@ = register value
273 * @void **ctx_out@ = where to put the @setvar@ context
274 * @void *ctx@ = context pointer
276 * Returns: @tvec_benchfindvar@ returns a pointer to the variable
277 * definition, or null; @setvar@ returns zero on success or
280 * Use: Find a definition for a special variable. The following
281 * special variables are supported.
283 * * %|@target|% is the (approximate) duration to run
286 * Unrecognized variables are passed to the subordinate
287 * environment, if there is one.
290 static int setvar(struct tvec_state
*tv
, const char *var
,
291 const union tvec_regval
*rv
, void *ctx
)
293 struct tvec_benchctx
*bc
= ctx
;
295 if (STRCMP(var
, ==, "@target")) {
296 if (bc
->f
&TVBF_SETTRG
) return (tvec_dupregerr(tv
, var
));
297 bc
->bst
->target_s
= rv
->f
; bc
->f
|= TVBF_SETTRG
;
298 } else assert("unknown var");
302 static const struct tvec_vardef target_var
=
303 { sizeof(struct tvec_reg
), setvar
, { "@target", &tvty_duration
, -1, 0 } };
305 const struct tvec_vardef
*tvec_benchfindvar
306 (struct tvec_state
*tv
, const char *var
, void **ctx_out
, void *ctx
)
308 struct tvec_benchctx
*bc
= ctx
;
309 const struct tvec_benchenv
*be
= bc
->be
;
310 const struct tvec_env
*subenv
= be
->env
;
312 if (STRCMP(var
, ==, "@target")) { *ctx_out
= bc
; return (&target_var
); }
313 else if (subenv
&& subenv
->findvar
)
314 return (subenv
->findvar(tv
, var
, ctx_out
, bc
->subctx
));
318 /* --- @tvec_benchbefore@ --- *
320 * Arguments: @struct tvec_state *tv@ = test vector state
321 * @void *ctx@ = context pointer
325 * Use: Invoke the subordinate environment's @before@ function to
326 * prepare for the benchmark.
329 void tvec_benchbefore(struct tvec_state
*tv
, void *ctx
)
331 struct tvec_benchctx
*bc
= ctx
;
332 const struct tvec_benchenv
*be
= bc
->be
;
333 const struct tvec_env
*subenv
= be
->env
;
335 if (subenv
&& subenv
->before
) subenv
->before(tv
, bc
->subctx
);
338 /* --- @tvec_benchrun@ --- *
340 * Arguments: @struct tvec_state *tv@ = test vector state
341 * @tvec_testfn *fn@ = test function to run
342 * @void *ctx@ = context pointer for the test function
346 * Use: Measures and reports the performance of a test function.
349 void tvec_benchrun(struct tvec_state
*tv
, tvec_testfn
*fn
, void *ctx
)
351 struct tvec_benchctx
*bc
= ctx
;
352 const struct tvec_benchenv
*be
= bc
->be
;
353 const struct tvec_env
*subenv
= be
->env
;
354 struct tvec_output
*o
= tv
->output
;
355 struct tvec_fallbackoutput fo
;
356 const struct tvec_benchoutops
*bo
;
358 struct bench_timing t
;
364 /* Decide on the kind of unit and the base count. */
366 if (be
->rbuf
< 0) unit
= BTU_OP
;
367 else { unit
= BTU_BYTE
; base
*= TVEC_REG(tv
, in
, be
->rbuf
)->v
.bytes
.sz
; }
369 /* Find the output operations and report the start. */
370 bo
= tvec_outputext(tv
, &o
, &fo
,
371 TVEC_BENCHOUTEXT
, &tvec_benchoutputfallback
);
372 bo
->bbench(o
, 0, unit
);
374 /* Run the benchmark. */
375 if (be
->riter
>= 0) {
376 n
= &TVEC_REG(tv
, in
, be
->riter
)->v
.u
;
377 if (subenv
&& subenv
->run
)
378 BENCH_MEASURE(bc
->bst
, rc
, &t
, base
)
379 { *n
= _bench_n
; subenv
->run(tv
, fn
, bc
->subctx
); }
381 BENCH_MEASURE(bc
->bst
, rc
, &t
, base
)
382 { *n
= _bench_n
; fn(tv
->in
, tv
->out
, bc
->subctx
); }
384 if (subenv
&& subenv
->run
)
385 BENCH_MEASURE(bc
->bst
, rc
, &t
, base
)
386 while (_bench_n
--) subenv
->run(tv
, fn
, bc
->subctx
);
388 BENCH_MEASURE(bc
->bst
, rc
, &t
, base
)
389 while (_bench_n
--) fn(tv
->in
, tv
->out
, bc
->subctx
);
392 /* Report the outcome. */
393 bo
->ebench(o
, 0, unit
, rc ?
0 : &t
);
396 /* --- @tvec_benchafter@ --- *
398 * Arguments: @struct tvec_state *tv@ = test vector state
399 * @void *ctx@ = context pointer
403 * Use: Invoke the subordinate environment's @after@ function to
404 * clean up after the benchmark.
407 void tvec_benchafter(struct tvec_state
*tv
, void *ctx
)
409 struct tvec_benchctx
*bc
= ctx
;
410 const struct tvec_benchenv
*be
= bc
->be
;
411 const struct tvec_env
*subenv
= be
->env
;
413 /* Restore the benchmark state's old target. */
414 if (bc
->bst
) bc
->bst
->target_s
= bc
->dflt_target
;
415 bc
->f
&= ~TVBF_SETTRG
;
417 /* Pass the call on to the subsidiary environment. */
418 if (subenv
&& subenv
->after
) subenv
->after(tv
, bc
->subctx
);
421 /* --- @tvec_benchteardown@ --- *
423 * Arguments: @struct tvec_state *tv@ = test vector state
424 * @void *ctx@ = context pointer
428 * Use: Tear down the benchmark environment.
431 void tvec_benchteardown(struct tvec_state
*tv
, void *ctx
)
433 struct tvec_benchctx
*bc
= ctx
;
434 const struct tvec_benchenv
*be
;
435 const struct tvec_env
*subenv
;
437 be
= bc
->be
; subenv
= be
->env
;
439 /* Tear down any subsidiary environment. */
440 if (subenv
&& subenv
->teardown
) subenv
->teardown(tv
, bc
->subctx
);
441 if (bc
->subctx
) x_free(tv
->a
, bc
->subctx
);
443 /* If the benchmark state was temporary, then dispose of it. */
445 if (be
->bst
) bc
->bst
->target_s
= bc
->dflt_target
;
446 else { bench_destroy(bc
->bst
); x_free(tv
->a
, bc
->bst
); }
450 /*----- That's all, folks -------------------------------------------------*/