3 .\" Manual for benchmarking core
5 .\" (c) 2024 Straylight/Edgeware
8 .\"----- Licensing notice ---------------------------------------------------
10 .\" This file is part of the mLib utilities library.
12 .\" mLib is free software: you can redistribute it and/or modify it under
13 .\" the terms of the GNU Library General Public License as published by
14 .\" the Free Software Foundation; either version 2 of the License, or (at
15 .\" your option) any later version.
17 .\" mLib is distributed in the hope that it will be useful, but WITHOUT
18 .\" ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 .\" FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
20 .\" License for more details.
22 .\" You should have received a copy of the GNU Library General Public
23 .\" License along with mLib. If not, write to the Free Software
24 .\" Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
27 .\"--------------------------------------------------------------------------
28 .so ../defs.man \" @@@PRE@@@
30 .\"--------------------------------------------------------------------------
31 .TH bench 3mLib "9 March 2024" "Straylight/Edgeware" "mLib utilities library"
32 .\" @bench_createtimer
38 .\"--------------------------------------------------------------------------
40 bench \- low-level benchmarking tools
42 .\"--------------------------------------------------------------------------
46 .B "#include <mLib/bench.h>"
49 .B "struct bench_time {"
52 .B " struct { kludge64 s; uint32 ns; } ts;"
59 .B "struct bench_timing {"
66 .B "#define BTF_T0 0u"
67 .B "#define BTF_T1 ..."
68 .B "struct bench_timerops {"
69 .BI " void (*describe)(struct bench_timer *" bt ", dstr *" d );
70 .BI " int (*preflight)(struct bench_timer *" bt );
71 .ta 2n +\w'\fBint (*now)('u
72 .BI " int (*now)(struct bench_timer *" bt ,
73 .BI " struct bench_time *" t_out ", unsigned " f );
74 .ta 2n +\w'\void (*diff)('u
75 .BI " void (*diff)(struct bench_timer *" bt ,
76 .BI " struct bench_timing *" delta_out ,
77 .BI " const struct bench_time *" t0 ,
78 .BI " const struct bench_time *" t1 );
79 .BI " void (*destroy)(struct bench_timer *" bt );
81 .B "struct bench_timer {"
82 .B " const struct bench_timerops *ops;"
85 .B "struct bench_state {"
87 .B " double target_s;"
91 .BI "typedef void bench_fn(unsigned long " n ", void *" ctx );
93 .B "#define BTF_TIMEOK ..."
94 .B "#define BTF_CYOK ..."
95 .B "#define BTF_CLB ..."
96 .B "#define BTF_ANY (BTF_TIMEOK | BTF_CYOK)"
98 .B "struct bench_timer *bench_createtimer(void);"
100 .BI "int bench_init(struct bench_state *" b ", struct bench_timer *" tm );
101 .BI "void bench_destroy(struct bench_state *" b );
102 .BI "int bench_calibrate(struct bench_state *" b );
103 .ta \w'\fBint bench_measure('u
104 .BI "int bench_measure(struct bench_state *" b ", struct bench_timing *" t_out ,
105 .BI " double " base ", bench_fn *" fn ", void *" ctx );
108 .\"--------------------------------------------------------------------------
113 provides declarations and defintions
114 for performing low-level benchmarks.
118 This function will be described in detail later,
120 it calls a caller-provided function,
121 instructing it to run adaptively chosen numbers of iterations,
122 in order to get a reasonably reliable measurement of its running time,
123 and then reports its results by filling in a structure.
125 With understanding this function as our objective,
126 we must examine all of the pieces involved in making it work.
128 .SS Timers in general
131 is a gadget which is capable of reporting the current time,
132 in seconds (ideally precise to tiny fractions of a second),
133 and/or in CPU cycles.
134 A timer is represented by a pointer to an object of type
135 .BR "struct bench_timer" .
136 This structure has a single member,
139 .BR "struct bench_timerops" ,
140 which is a table of function pointers;
141 typically, a timer has more data following this,
142 but this fact is not exposed to applications.
144 The function pointers in
145 .B "struct bench_timerops"
150 must always point to the timer object itself.
152 .IB tm ->ops->describe( tm ", " d)
153 Write a description of the timer to the dynamic string
156 .IB tm ->ops->preflight( tm )
157 Ensure that the timer is in working order,
158 and perform any necessary per-thread or per-process setup.
161 function is likely to work properly
162 when called from the same thread
164 otherwise return \-1.
166 .IB tm ->ops->now( tm ", " t_out ", " f )
167 Store the current time in
173 to indicate that this is the second call in a pair;
174 leave it clear for the first call.
177 flag is defined to be zero for symmetry.)
178 Return zero on success
181 return \-1 if timing failed but
182 trying again immediately has a reasonable chance of success.
184 .IB tm ->ops->diff( tm ", " delta_out ", " t0 ", " t1 )
187 the difference between the two times
192 .IB tm ->ops->destroy( tm )
194 releasing all of the resources that it holds.
198 structure reports the difference between two times,
199 as determined by a timer's
207 is set if the passage-of-time measurement in
211 is set if the cycle count in
224 if the timer returned any valid timing information.
227 The number of iterations performed by the benchmark function
228 on its satisfactory run,
233 The time taken for the satisfactory run of the benchmark function,
241 The number of CPU cycles used
242 in the satisfactory run of the benchmark function,
250 .B "struct bench_time"
251 represents a single instant in time,
252 as captured by a timer's
255 The use of this structure is a private matter for the timer:
256 the only hard requirement is that the
258 function should be able to compute the difference between two times.
259 However, the intent is that
260 a passage-of-time measurement is stored in the
263 a cycle count is stored in the
272 if the passage-of-time or cycle count respectively are valid.
274 .SS The built-in timer
277 constructs and returns a timer.
278 It takes a single argument,
281 from which it reads configuration information.
284 fails, it returns a null pointer.
288 pointer may safely be null,
289 in which case a default configuration will be used.
292 set this pointer to a value supplied by a user,
293 e.g., through a command-line argument,
294 environment variable, or
297 The built-in timer makes use of one or two
299 a `clock' subtimer to measure the passage of time,
300 and possibly a `cycle' subtimer to count CPU cycles.
302 The configuration string consists of a sequence of words
303 separated by whitespace.
304 There may be additional whitespace at the start and end of the string.
305 The words recognized are as follows.
308 Prints a list of the available clock and cycle subtimers
312 Use the first of the listed clock subtimers
313 to initialize successfully
314 as the clock subtimer.
315 If none of the subtimers can be initialized,
316 then construction of the timer as a whole fails.
319 Use the first of the listed subtimers
320 to initialize successfully
321 as the cycle subtimer.
322 If none of the subtimers can be initialized,
323 then construction of the timer as a whole fails.
325 The clock subtimers are as follows.
326 Not all of them will be available on every platform.
328 .B linux-x86-perf-rdpmc-hw-cycles
329 This is a dummy companion to the similarly named cycle subtimer;
330 see its description below.
332 .B posix-thread-cputime
333 Measures the passage of time using
334 .BR clock_gettime (2),
336 .B CLOCK_\%THREAD_\%CPUTIME_\%ID
340 Measures the passage of time using
344 is part of the original ANSI\ C standard,
345 this subtimer should always be available.
346 However, it may produce unhelpful results
347 if other threads are running.
349 The cycle subtimers are as follows.
350 Not all of them will be available on every platform.
352 .B linux-perf-read-hw-cycles
353 Counts CPU cycles using the Linux-specific
354 .BR perf_event_open (2)
356 .BR PERF_\%COUNT_\%HW_\%CPU_\%CYCLES
358 Only available on Linux.
359 It will fail to initialize
360 if access to performance counters is restricted,
362 .B /proc/sys/kernel/perf_event_paranoid
365 .B linux-perf-rdpmc-hw-cycles
366 Counts CPU cycles using the Linux-specific
367 .BR perf_event_open (2)
370 .B linux-x86-perf-read-hw-cycles
372 except that it additionally uses the i386/AMD64
377 together with information provided by the kernel
378 through a memory-mapped page
379 to do its measurements without any system call overheads.
380 It does passage-of-time and cycle counting in a single operation,
381 so no separate clock subtimer is required:
382 the similarly-named clock subtimer does nothing
383 except check that the
384 .B linux-x86-perf-rdpmc-hw-cycles
385 cycle subtimer has been selected.
386 This is almost certainly the best choice if it's available.
389 Counts CPU cycles using the x86
392 This instruction is not really suitable for performance measurement:
393 it gives misleading results on CPUs with variable clock frequency.
396 Counts CPU cycles using the x86
399 This has the downsides of
402 but also fails to detect when the thread has been suspended
403 or transferred to a different CPU core
404 and gives misleading answers in this case.
405 Not really recommended.
408 A dummy cycle counter,
409 which will initialize successfully
410 and then fail to report cycle counts.
411 This is a reasonable fallback in many situations.
413 The built-in preference order for clock subtimers,
414 from most to least preferred, is
415 .BR linux-x86-perf-rdpmc-hw-cycles ,
417 .BR posix-thread-cputime ,
420 The built-in preference order for cycle subtimers,
421 from most to least preferred, is
422 .BR linux-x86-perf-rdpmc-hw-cycles
424 .BR linux-x86-perf-read-hw-cycles ,
432 .SS The benchmark state
435 tracks the information needed to measure performance of functions.
436 It is represented by a
437 .B struct bench_state
440 The benchmark state is initialized by calling
442 passing the address of the state structure to be initialized,
443 and a pointer to a timer.
446 is called with a non-null timer pointer,
447 then it will not fail;
448 the benchmark state will be initialized,
449 and the function returns zero.
450 If the timer pointer is null,
453 attempts to construct a timer for itself
455 .BR bench_createtimer .
457 then the benchmark state will be initialized,
458 and the function returns zero.
460 the timer becomes owned by the benchmark state:
463 on the benchmark state will destroy the timer.
466 is called with a null timer pointer,
467 and its attempt to create a timer for itself fails,
471 the benchmark state is not initialized
472 and can safely be discarded;
476 on the unsuccessfully benchmark state is safe and has no effect.
481 releases any resources it holds,
482 most notably its timer, if any.
485 .B struct bench_state
486 is defined in the header file,
487 only two members are available for use by applications.
490 A word containing flags.
493 The target time for which to try run a benchmark, in seconds.
494 After initialization, this is set to 1.0,
495 though applications can override it.
497 Before the benchmark state can be used in measurements,
500 This is performed by calling
502 on the benchmark state.
503 Calibration takes a noticeable amount of time
504 (currently about 0.25\*,s),
505 so it makes sense to defer it until it's known to be necessary.
507 Calibration is carried out separately, but in parallel,
508 for the timer's passage-of-time measurement and cycle counter.
509 Either or both of these calibrations can succeed or fail;
510 if passage-of-time calibration fails,
511 then cycle count calibration is impossible.
515 sets flag in the benchmark state's
518 if passage-of-time calibration succeeded,
521 if cycle-count calibration succeeded,
526 is set unconditionally,
527 as a persistent indication that calibration has been attempted.
531 function returns zero if it successfully calibrated
532 at least the passage-of-time measurement;
533 otherwise, it returns \-1.
536 is called for a second or subsequent time on the same benchmark state,
537 it returns immediately,
538 either returning 0 or \-1
539 according to whether passage-of-time had previously been calibrated.
543 .I benchmark function
546 .BI "void " fn "(unsigned long " n ", void *" ctx );
548 When called, it should perform the operation to be measured
553 argument is a pointer passed into
555 for the benchmark function's own purposes.
559 receives five arguments.
562 points to the benchmark state to be used.
566 .BR struct bench_timing
567 in which the measurement should be left.
568 This structure is described below.
571 is a count of the number of operations performed
572 by each iteration of the benchmark function.
575 is a benchmark function, described above.
578 is a pointer to be passed to the benchmark function.
580 does not interpret this pointer in any way.
584 function calls its benchark function repeatedly
585 with different iteration counts
587 with the objective that the call take approximately
589 seconds, as established in the benchmark state.
596 is satisfied when a call takes at least
598 Once the function finds a satisfactory number of iterations,
599 it stores the results in
601 If measurement succeeds, then
605 most likely because the timer failed \(en
608 .\"--------------------------------------------------------------------------
614 .\"--------------------------------------------------------------------------
617 Mark Wooding, <mdw@distorted.org.uk>
619 .\"----- That's all, folks --------------------------------------------------