Skip to content

Commit 023209e

Browse files
author
Jeff Treuting
committed
RunTests for a total amount of time
2 new overloads for RunTests: RunTests(TimeSpan) RunTests(int mins, int secs) In order to still run it in various orders it runs batches and each batch is in a random order. It starts with a batch size of 1 and then based on how long that takes it estimates a new batch size in order to get approx 1 batch for each test. So if you are running 4 different tests, it should try to do this in 4 batches to give each test a chance to go first.
1 parent 67baa15 commit 023209e

3 files changed

Lines changed: 84 additions & 14 deletions

File tree

‎SharpBenchmark.Samples/Code/Sample2.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ public override void Execute(string[] args)
1313
AddTest("Activator.CreateInstance<T>()", () => ActivatorCreateInstanceGeneric<TestClass>());
1414
AddTest("Activator.CreateInstance(type)", () => ActivatorCreateInstanceType(typeof (TestClass)));
1515

16-
RunTests(2000000);
16+
RunTests(0, 10);
1717
}
1818

1919
private static TestClass NewTestClass()

‎SharpBenchmark.Samples/Sample.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,16 @@ public void RunTests(int num)
2121
Benchmarker.RunTests(num);
2222
}
2323

24+
public void RunTests(TimeSpan totalTime)
25+
{
26+
Benchmarker.RunTests(totalTime);
27+
}
28+
29+
public void RunTests(int minutes, int seconds)
30+
{
31+
Benchmarker.RunTests(minutes, seconds);
32+
}
33+
2434
public abstract void Execute(string[] args);
2535
}
2636
}

‎SharpBenchmark/Benchmarker.cs‎

Lines changed: 73 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ public class Benchmarker
1414
private readonly TextWriter _writer;
1515
private StringBuilder _builder;
1616
private readonly string _fileOutputPath = null;
17+
private double _totalMilliseconds = 0;
1718

1819
private readonly IDictionary<string, BenchmarkResult> _results = new Dictionary<string, BenchmarkResult>();
1920

@@ -42,36 +43,94 @@ public void AddTest(string title, Action test)
4243
_tests.Add(new BenchmarkItem { Title = title, Test = test });
4344
}
4445

46+
public void RunTests(TimeSpan totalTime)
47+
{
48+
RunWarmUp(1);
49+
50+
InternalRun(null, totalTime);
51+
}
52+
53+
public void RunTests(int minutes, int seconds)
54+
{
55+
RunTests(TimeSpan.FromSeconds((minutes * 60) + seconds));
56+
}
57+
4558
public void RunTests(int num = 100000)
4659
{
47-
_builder = new StringBuilder();
60+
RunWarmUp(num);
61+
62+
InternalRun(num, null);
63+
}
4864

65+
private void RunWarmUp(int totalNum)
66+
{
4967
// WARM-UP
5068
// run thru each of them once because otherwise the first loop is slower due to the Just In Time compilation
51-
var numWarmups = Math.Ceiling(num * WarmUpPercentage);
52-
53-
// WriteLine();
54-
// WriteLine(String.Format("Running warmup tests {0:#,0} times", numWarmups));
69+
var numWarmups = Math.Ceiling(totalNum * WarmUpPercentage);
5570

56-
for (var i = 0; i < numWarmups; i++ )
71+
for (var i = 0; i < numWarmups; i++)
72+
{
5773
foreach (var test in _tests)
5874
{
5975
test.Test();
6076
}
77+
}
78+
}
79+
80+
private void InternalRun(int? totalTests, TimeSpan? totalTime)
81+
{
82+
_builder = new StringBuilder();
6183

6284
WriteLine();
63-
WriteLine(String.Format("Running each test {0:#,0} times", num));
85+
86+
if (totalTests.HasValue)
87+
{
88+
WriteLine(String.Format("Running each test {0:#,0} times", totalTests));
89+
}
90+
else
91+
{
92+
WriteLine(String.Format("Running tests for {0}", totalTime));
93+
}
94+
6495
WriteLine();
6596
WriteLine("----------------------------------------------------");
6697
WriteLine("Running tests: ");
6798

68-
for (var i = 0; i < _tests.Count; i++)
99+
if (totalTests.HasValue)
69100
{
70-
WriteLine(String.Format(" Pass {0}...", i+1));
71-
InternalRun(num / _tests.Count);
101+
for (var i = 0; i < _tests.Count; i++)
102+
{
103+
WriteLine(String.Format(" Pass {0}...", i + 1));
104+
InternalSingleRun(totalTests.Value/_tests.Count);
105+
}
72106
}
73-
107+
else if (totalTime.HasValue)
108+
{
109+
// the batch size will get adjusted based on how long they are taking to process
110+
// we need to start with 1 and then increase based on their speed
111+
var batchSize = 1;
112+
long totalTimesRan = 0;
74113

114+
while (_totalMilliseconds < totalTime.Value.TotalMilliseconds)
115+
{
116+
InternalSingleRun(batchSize);
117+
totalTimesRan += batchSize;
118+
119+
// adjust batch size based on the current pace
120+
var percentDone = _totalMilliseconds/totalTime.Value.TotalMilliseconds;
121+
var estimatedMoreRuns = totalTimesRan/percentDone;
122+
123+
// let's be safe and try to split it up into a batch for each test so it gets to run in various order
124+
batchSize = (int)Math.Floor(estimatedMoreRuns / _tests.Count+1);
125+
126+
WriteLine(String.Format(" Running new batch of {0:#,0} tests", batchSize));
127+
}
128+
129+
WriteLine();
130+
WriteLine(String.Format(" Ran {0:#,0} times in time allotted", totalTimesRan));
131+
WriteLine(String.Format(" Took a total of {0}", TimeSpan.FromMilliseconds(_totalMilliseconds)));
132+
}
133+
75134
DisplayResults();
76135

77136
var resultsText = _builder.ToString();
@@ -90,10 +149,10 @@ public void RunTests(int num = 100000)
90149
}
91150
}
92151

93-
private void InternalRun(int num)
152+
private void InternalSingleRun(int num)
94153
{
95154
var sw = new Stopwatch();
96-
155+
97156
foreach (var benchmarkItem in _tests.OrderBy(x => Guid.NewGuid()))
98157
{
99158
sw.Reset();
@@ -132,6 +191,7 @@ private void AddResult(BenchmarkItem item, double totalTime, int timesRun)
132191
var result = _results[item.Title];
133192
result.TimeRun += timesRun;
134193
result.TotalTime += totalTime;
194+
_totalMilliseconds += totalTime;
135195
}
136196

137197

0 commit comments

Comments
 (0)