Skip to content

Commit 6d8687d

Browse files
runrevmarkrunrevmark
committed
[[ Benchmarks ]] A simple benchmark system.
Modelled on the livecodescript unit test runner, this adds a simple benchmarking suite execution system. Just like the unit test system, benchmarks are split up into script files which contains handlers starting with 'Benchmark'. Each such handler should implement a single benchmark. A benchmark starts timing by using BenchmarkStartTiming and finishes timing by using BenchmarkEndTiming. The time taken is then emitted to stdout to be picked up by the host. The host can execute many benchmarks in one go. It outputs a _benchmark_suite.log file, containing one benchmark per line. Each line contains two tab-separated columns - the first is the benchmark name, the second is the time taken in milliseconds.
1 parent fb2e63d commit 6d8687d

File tree

3 files changed

+406
-0
lines changed

3 files changed

+406
-0
lines changed

‎benchmarks/_benchmarklib.livecodescript‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
script "BenchmarkLibrary"
2+
/*
3+
Copyright (C) 2015 LiveCode Ltd.
4+
5+
This file is part of LiveCode.
6+
7+
LiveCode is free software; you can redistribute it and/or modify it under
8+
the terms of the GNU General Public License v3 as published by the Free
9+
Software Foundation.
10+
11+
LiveCode is distributed in the hope that it will be useful, but WITHOUT ANY
12+
WARRANTY; without even the implied warranty of MERCHANTABILITY or
13+
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14+
for more details.
15+
16+
You should have received a copy of the GNU General Public License
17+
along with LiveCode. If not see <http://www.gnu.org/licenses/>. */
18+
19+
----------------------------------------------------------------
20+
-- Helper functions
21+
----------------------------------------------------------------
22+
23+
----------------------------------------------------------------
24+
-- Benchmark library functions
25+
----------------------------------------------------------------
26+
27+
local sBenchmarkStartTime
28+
29+
on BenchmarkStartTiming
30+
put the millisecs into sBenchmarkStartTime
31+
end BenchmarkStartTiming
32+
33+
on BenchmarkEndTiming
34+
write (the millisecs - sBenchmarkStartTime) to stdout
35+
end BenchmarkEndTiming
36+
37+
on errorDialog executionError, parseError
38+
write executionError & return to stderr
39+
quit 1
40+
end errorDialog

‎benchmarks/_benchmarkrunner.livecodescript‎

Lines changed: 334 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,334 @@
1+
script "BenchmarkRunner"
2+
/*
3+
Copyright (C) 2015 LiveCode Ltd.
4+
5+
This file is part of LiveCode.
6+
7+
LiveCode is free software; you can redistribute it and/or modify it under
8+
the terms of the GNU General Public License v3 as published by the Free
9+
Software Foundation.
10+
11+
LiveCode is distributed in the hope that it will be useful, but WITHOUT ANY
12+
WARRANTY; without even the implied warranty of MERCHANTABILITY or
13+
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14+
for more details.
15+
16+
You should have received a copy of the GNU General Public License
17+
along with LiveCode. If not see <http://www.gnu.org/licenses/>. */
18+
19+
-- FIXME provide this on the command line
20+
constant kLogFilename = "_benchmark_suite.log"
21+
22+
-- This is the message dispatched before invoking each benchmark command
23+
constant kSetupMessage = "BenchmarkSetup"
24+
-- And this message is dispatched *after* each benchmark command
25+
constant kTeardownMessage = "BenchmarkTearDown"
26+
27+
on startup
28+
send "BenchmarkRunnerMain" to me in 0
29+
end startup
30+
31+
----------------------------------------------------------------
32+
-- Command-line processing
33+
----------------------------------------------------------------
34+
35+
private function getCommandLineInfo
36+
local tRawArg, tSelfCommand, tSelfScript, tInArgs, tArgs
37+
38+
put false into tInArgs
39+
40+
-- Treat everything up to & including the first
41+
-- ".livecodescript" file as the command for running the test
42+
-- runner, and everything after it as test runner arguments
43+
put the commandName into tSelfCommand[1]
44+
repeat for each element tRawArg in the commandArguments
45+
46+
if tInArgs then
47+
put tRawArg into tArgs[1 + the number of elements in tArgs]
48+
else
49+
put tRawArg into tSelfCommand[1 + the number of elements in tSelfCommand]
50+
if tRawArg ends with ".livecodescript" then
51+
put tRawArg into tSelfScript
52+
put true into tInArgs
53+
end if
54+
end if
55+
56+
end repeat
57+
58+
local tInfo
59+
put tSelfCommand into tInfo["self-command"]
60+
put tSelfScript into tInfo["self-script"]
61+
put tArgs into tInfo["args"]
62+
63+
return tInfo
64+
end getCommandLineInfo
65+
66+
----------------------------------------------------------------
67+
-- Top-level actions
68+
----------------------------------------------------------------
69+
70+
command BenchmarkRunnerMain
71+
local tInfo
72+
put getCommandLineInfo() into tInfo
73+
74+
switch tInfo["args"][1]
75+
case "invoke"
76+
doInvoke tInfo
77+
break
78+
case "run"
79+
doRun tInfo
80+
break
81+
case "--help"
82+
case "-h"
83+
case "help"
84+
doUsage 0
85+
break
86+
default
87+
doUsage 1
88+
break
89+
end switch
90+
quit 0
91+
end BenchmarkRunnerMain
92+
93+
private command doInvoke pInfo
94+
put pInfo["args"][2] into pInfo["invoke"]["script"]
95+
put pInfo["args"][3] into pInfo["invoke"]["command"]
96+
97+
invokeLoadLibrary pInfo
98+
invokeBenchmark pInfo
99+
end doInvoke
100+
101+
private command doRun pInfo
102+
local tScript, tCommand, tLog
103+
put pInfo["args"][2] into tScript
104+
put pInfo["args"][3] into tCommand
105+
if tScript is empty then
106+
runAllScripts pInfo
107+
else if tCommand is empty then
108+
runSingleScript pInfo, tScript
109+
else
110+
runSingleCommand pInfo, tScript, tCommand
111+
end if
112+
113+
put the result into tLog
114+
115+
-- Save the log to file
116+
local tLogToWrite
117+
put tLog into tLogToWrite
118+
if the platform is "win32" then
119+
replace return with numToChar(13) & numToChar(10) in tLogToWrite
120+
end if
121+
put tLogToWrite into url ("binfile:" & kLogFilename)
122+
end doRun
123+
124+
private command doUsage pStatus
125+
write "Usage: _benchmarkrunner.livecodescript run [SCRIPT [COMMAND]]" & return to stderr
126+
quit pStatus
127+
end doUsage
128+
129+
on errorDialog pExecutionError
130+
write "ERROR:" && pExecutionError & return to stderr
131+
quit 1
132+
end errorDialog
133+
134+
----------------------------------------------------------------
135+
-- Support for invoking test commands
136+
----------------------------------------------------------------
137+
138+
-- Execute a benchmark
139+
private command invokeBenchmark pInfo
140+
local tStackName
141+
142+
-- This should auto-load the test script
143+
put the name of stack pInfo["invoke"]["script"] into tStackName
144+
145+
-- Dispatch the test setup command, and the test command itself
146+
dispatch kSetupMessage to tStackName
147+
dispatch pInfo["invoke"]["command"] to tStackName
148+
dispatch kTeardownMessage to tStackName
149+
end invokeBenchmark
150+
151+
-- Add the unit test library stack to the backscripts
152+
private command invokeLoadLibrary pInfo
153+
local tStackName, tStackFile
154+
155+
-- This should auto-load the library
156+
put invokeGetLibraryStack(pInfo) into tStackFile
157+
put the name of stack tStackFile into tStackName
158+
159+
-- Add the library to the backscripts
160+
insert the script of stack tStackName into back
161+
end invokeLoadLibrary
162+
163+
-- Return the filename of the unit test library stack
164+
private function invokeGetLibraryStack pInfo
165+
local tFilename
166+
put pInfo["self-script"] into tFilename
167+
168+
set the itemDelimiter to slash
169+
put "_benchmarklib.livecodescript" into item -1 of tFilename
170+
171+
return tFilename
172+
end invokeGetLibraryStack
173+
174+
----------------------------------------------------------------
175+
-- Support for running tests
176+
----------------------------------------------------------------
177+
178+
-- Run all the benchmark scripts that can be found below the current
179+
-- directory
180+
private command runAllScripts pInfo
181+
local tFile, tLog
182+
repeat for each element tFile in runGetBenchmarkFileNames()
183+
runBenchmarkScript pInfo, tFile
184+
put the result after tLog
185+
end repeat
186+
187+
return tLog
188+
end runAllScripts
189+
190+
-- Run the benchmarks found in one specific script file
191+
private command runSingleScript pInfo, pScriptFile
192+
local tCommand, tLog, tMetadata
193+
194+
repeat for each element tCommand in runGetBenchmarkCommandNames(pScriptFile)
195+
runSingleCommand pInfo, pScriptFile, tCommand
196+
put the result after tLog
197+
end repeat
198+
199+
return tLog
200+
end runSingleScript
201+
202+
-- Run a specific named benchmark command tCommand in a script file
203+
-- tScriptFile
204+
private command runSingleCommand pInfo, pScriptFile, pCommand
205+
local tArg, tCommandLine
206+
207+
write "Running " & pCommand & "... " to stdout
208+
209+
repeat for each element tArg in pInfo["self-command"]
210+
put tArg & " " after tCommandLine
211+
end repeat
212+
213+
put "invoke" && pScriptFile && pCommand after tCommandLine
214+
215+
-- Invoke the test in a subprocess. This ensures that we can detect
216+
-- if a crash occurs
217+
local tBenchmarkTime, tBenchmarkExitStatus
218+
put shell(tCommandLine) into tBenchmarkTime
219+
put the result into tBenchmarkExitStatus
220+
221+
-- Check the exit status.
222+
if tBenchmarkExitStatus is not empty then
223+
put "failed" into tBenchmarkTime
224+
end if
225+
226+
write tBenchmarkTime & " ms" & return to stdout
227+
228+
return pCommand & tab & tBenchmarkTime & return
229+
end runSingleCommand
230+
231+
-- Get all livecode script files beneath the CWD, apart from
232+
-- filenames starting with "." or "_"
233+
private function runGetBenchmarkFileNames
234+
local tFiles, tCount
235+
236+
put empty into tFiles
237+
put 0 into tCount
238+
239+
runGetBenchmarkFileNames_Recursive the defaultfolder, empty, tFiles, tCount
240+
241+
return tFiles
242+
end runGetBenchmarkFileNames
243+
244+
-- Helper command used by runGetTestFileNames
245+
private command runGetBenchmarkFileNames_Recursive pPath, pRelPath, @xFiles, @xCount
246+
-- Save the CWD
247+
local tSaveFolder
248+
put the defaultfolder into tSaveFolder
249+
set the defaultfolder to pPath
250+
251+
-- Process files in the current directory
252+
local tFile
253+
repeat for each line tFile in the files
254+
if tFile ends with ".livecodescript" and \
255+
not (tFile begins with "." or tFile begins with "_") then
256+
257+
if pRelPath is not empty then
258+
put pRelPath & slash before tFile
259+
end if
260+
261+
add 1 to xCount
262+
put tFile into xFiles[xCount]
263+
end if
264+
end repeat
265+
266+
-- Process subdirectories
267+
local tFolder, tFolderPath
268+
repeat for each line tFolder in the folders
269+
if tFolder begins with "." then
270+
next repeat
271+
end if
272+
273+
put pPath & slash & tFolder into tFolderPath
274+
275+
if pRelPath is not empty then
276+
put pRelPath & slash before tFolder
277+
end if
278+
279+
runGetBenchmarkFileNames_Recursive tFolderPath, tFolder, xFiles, xCount
280+
end repeat
281+
282+
-- Restore the CWD
283+
set the defaultfolder to tSaveFolder
284+
end runGetBenchmarkFileNames_Recursive
285+
286+
-- Get a number-indexed array contain the names of all "test"
287+
-- commands in pFilename.
288+
private function runGetBenchmarkCommandNames pFilename
289+
local tScript
290+
291+
-- Get the contents of the file
292+
open file pFilename for "UTF-8" text read
293+
if the result is not empty then
294+
throw the result
295+
end if
296+
297+
read from file pFilename until end
298+
put it into tScript
299+
300+
close file pFilename
301+
302+
-- Scan the file for "on Benchmark*" definitions
303+
local tCommandNames, tCount, tLine, tName
304+
305+
repeat for each line tLine in tScript
306+
if token 1 of tLine is not "on" then
307+
next repeat
308+
end if
309+
310+
put token 2 of tLine into tName
311+
312+
if not (tName begins with "Benchmark") then
313+
next repeat
314+
end if
315+
316+
-- Exclude the test setup message
317+
if tName is kSetupMessage or tName is kTeardownMessage then
318+
next repeat
319+
end if
320+
321+
add 1 to tCount
322+
put tName into tCommandNames[tCount]
323+
end repeat
324+
325+
return tCommandNames
326+
end runGetBenchmarkCommandNames
327+
328+
-- Prettify a test name by removing a ".livecodescript" suffix
329+
private function runGetPrettyBenchmarkName pFilename
330+
if pFilename ends with ".livecodescript" then
331+
set the itemDelimiter to "."
332+
return item 1 to -2 of pFileName
333+
end if
334+
end runGetPrettyBenchmarkName

0 commit comments

Comments
 (0)