Skip to content

Commit ed7d1cf

Browse files
tboschtbosch
committed
feat(perf): cloud reporter, more generic table layout
1 parent 0aa0c26 commit ed7d1cf

5 files changed

Lines changed: 101 additions & 51 deletions

File tree

‎protractor-perf-shared.js‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,30 @@
11
var config = exports.config = require('./protractor-shared.js').config;
22
// load traceur runtime as our tests are written in es6
33
require('traceur/bin/traceur-runtime.js');
4+
var nodeUuid = require('node-uuid');
45

56
var cloudReporterConfig;
67
if (process.env.CLOUD_SECRET_PATH) {
78
console.log('using cloud reporter!');
89
cloudReporterConfig = {
910
auth: require(process.env.CLOUD_SECRET_PATH),
1011
projectId: 'angular-perf',
11-
datasetId: 'benchmarks'
12+
datasetId: 'benchmarks',
13+
tableId: 'ng2perf'
1214
};
1315
}
1416

1517
config.specs = ['dist/cjs/**/*_perf.js'];
1618
config.jasmineNodeOpts.defaultTimeoutInterval = 80000;
1719

20+
var runId = nodeUuid.v1();
21+
if (process.env.GIT_SHA) {
22+
runId = process.env.GIT_SHA + ' ' + runId;
23+
}
24+
1825
config.params = {
1926
benchmark: {
27+
runId: runId,
2028
// size of the sample to take
2129
sampleSize: 20,
2230
timeout: 60000,

‎scripts/jenkins/jenkins_perf.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ export CHANNEL=stable
1010
export ARCH=macos-ia32
1111
export PERF_BROWSERS=ChromeAndroid
1212
export CLOUD_SECRET_PATH="/Users/Shared/jenkins/keys/perf-cloud-secret"
13+
export GIT_SHA=$(git rev-parse HEAD)
1314

1415
nvm use 0.10
1516

‎tools/benchpress/src/benchmark.es6‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
var statistics = require('./statistics');
22
var commands = require('./commands');
3-
var nodeUuid = require('node-uuid');
43
var webdriver = require('protractor/node_modules/selenium-webdriver');
54

65
var SUPPORTED_METRICS = {
@@ -21,11 +20,10 @@ module.exports = {
2120
};
2221

2322
function runBenchmark(config, workCallback) {
24-
var sampleId = nodeUuid.v1();
2523
var reporters = config.reporters.filter(function(Class) {
2624
return !!Class;
2725
}).map(function(Class) {
28-
return new Class(sampleId, config);
26+
return new Class(config);
2927
});
3028
var scriptMetricIndex = -1;
3129
config.metrics.forEach(function(metric, index) {

‎tools/benchpress/src/cloud_reporter.es6‎

Lines changed: 88 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,16 @@ var google = require('googleapis');
22
var bigquery = google.bigquery('v2');
33
var webdriver = require('protractor/node_modules/selenium-webdriver');
44

5-
var HEADER_FIELDS = [
5+
var TABLE_FIELDS = [
66
{
77
"name": 'runId',
88
"type": 'STRING',
9-
"description": 'uuid for the benchmark run'
9+
"description": 'git SHA and uuid for the benchmark run'
10+
},
11+
{
12+
"name": 'benchmarkId',
13+
"type": 'STRING',
14+
"description": 'id of the benchmark'
1015
},
1116
{
1217
"name": 'index',
@@ -26,22 +31,65 @@ var HEADER_FIELDS = [
2631
"name": 'forceGc',
2732
"type": 'BOOLEAN',
2833
"description": 'whether gc was forced at end of action'
34+
},
35+
{
36+
"name": 'stable',
37+
"type": 'BOOLEAN',
38+
"description": 'whether this entry was part of the stable sample'
39+
},
40+
{
41+
"name": 'params',
42+
"type": 'RECORD',
43+
"description": 'parameters of the benchmark',
44+
"mode": 'REPEATED',
45+
"fields": [
46+
{
47+
"name": 'name',
48+
"type": 'STRING',
49+
"description": 'param name'
50+
},
51+
{
52+
"name": 'strvalue',
53+
"type": 'STRING',
54+
"description": 'param value for strings'
55+
},
56+
{
57+
"name": 'numvalue',
58+
"type": 'FLOAT',
59+
"description": 'param value for numbers'
60+
}
61+
]
62+
},
63+
{
64+
"name": 'metrics',
65+
"type": 'RECORD',
66+
"description": 'metrics of the benchmark',
67+
"mode": 'REPEATED',
68+
"fields": [
69+
{
70+
"name": 'name',
71+
"type": 'STRING',
72+
"description": 'metric name'
73+
},
74+
{
75+
"name": 'value',
76+
"type": 'FLOAT',
77+
"description": 'metric value'
78+
}
79+
]
2980
}
3081
];
3182

3283
class CloudReporter {
33-
constructor(runId, benchmarkConfig) {
34-
this.stableRowsTableConfig = createTableConfig(benchmarkConfig, '_stable');
35-
this.allRowsTableConfig = createTableConfig(benchmarkConfig, '_all')
84+
constructor(benchmarkConfig) {
85+
this.tableConfig = createTableConfig(benchmarkConfig);
3686
this.authConfig = benchmarkConfig.cloudReporter.auth;
3787
this.benchmarkConfig = benchmarkConfig;
38-
this.runId = runId;
39-
this.allRows = [];
88+
this.allSample = [];
4089
var self = this;
4190
browser.executeScript('return navigator.userAgent').then(function(userAgent) {
4291
self.browserUserAgent = userAgent;
4392
});
44-
4593
}
4694
begin() {
4795
var self = this;
@@ -52,66 +100,62 @@ class CloudReporter {
52100
});
53101
});
54102
flow.execute(function() {
55-
return webdriver.promise.all([
56-
getOrCreateTable(self.authClient, self.allRowsTableConfig),
57-
getOrCreateTable(self.authClient, self.stableRowsTableConfig)
58-
]);
103+
return getOrCreateTable(self.authClient, self.tableConfig);
59104
});
60105
}
61106
add(data) {
62-
this.allRows.push(this._convertToTableRow(data));
107+
this.allSample.push(data);
63108
}
64109
end(stableSample) {
65110
var self = this;
66111
var flow = browser.driver.controlFlow();
67-
var stableRows = stableSample.map(function(data) {
68-
return self._convertToTableRow(data);
112+
var allRows = this.allSample.map(function(data) {
113+
return self._convertToTableRow(data, stableSample);
69114
});
70115
flow.execute(function() {
71-
return webdriver.promise.all([
72-
insertRows(self.authClient, self.stableRowsTableConfig, stableRows),
73-
insertRows(self.authClient, self.allRowsTableConfig, self.allRows)
74-
]);
116+
return insertRows(self.authClient, self.tableConfig, allRows)
75117
});
76118
}
77-
_convertToTableRow(benchpressRow) {
119+
_convertToTableRow(benchpressRow, stableSample) {
78120
var tableRow = {
79-
runId: this.runId,
121+
runId: this.benchmarkConfig.runId,
122+
benchmarkId: this.benchmarkConfig.id,
80123
index: benchpressRow.index,
81124
creationTime: new Date(),
82125
browser: this.browserUserAgent,
83-
forceGc: benchpressRow.forceGc
126+
forceGc: benchpressRow.forceGc,
127+
stable: stableSample.indexOf(benchpressRow) >= 0,
128+
params: this.benchmarkConfig.params.map(function(param) {
129+
if (typeof param.value === 'number') {
130+
return {
131+
name: param.name,
132+
numvalue: param.value
133+
};
134+
} else {
135+
return {
136+
name: param.name,
137+
strvalue: ''+param.value
138+
}
139+
}
140+
}),
141+
metrics: this.benchmarkConfig.metrics.map(function(metricName, index) {
142+
return {
143+
name: metricName,
144+
value: benchpressRow.values[index]
145+
};
146+
})
84147
};
85-
this.benchmarkConfig.params.forEach(function(param) {
86-
tableRow['p_'+param.name] = param.value;
87-
});
88-
this.benchmarkConfig.metrics.forEach(function(metric, index) {
89-
tableRow['m_'+metric] = benchpressRow.values[index];
90-
});
91148
return tableRow;
92149
}
93150
}
94151

95-
function createTableConfig(benchmarkConfig, tableSuffix) {
96-
var tableId = (benchmarkConfig.id+tableSuffix).replace(/\./g, '_');
152+
function createTableConfig(benchmarkConfig) {
97153
return {
98154
projectId: benchmarkConfig.cloudReporter.projectId,
99155
datasetId: benchmarkConfig.cloudReporter.datasetId,
100156
table: {
101-
id: tableId,
102-
fields: HEADER_FIELDS
103-
.concat(benchmarkConfig.params.map(function(param) {
104-
return {
105-
"name": 'p_'+param.name,
106-
"type": 'FLOAT'
107-
};
108-
}))
109-
.concat(benchmarkConfig.metrics.map(function(metricName) {
110-
return {
111-
"name": 'm_'+metricName,
112-
"type": 'FLOAT'
113-
};
114-
}))
157+
id: benchmarkConfig.cloudReporter.tableId,
158+
fields: TABLE_FIELDS
115159
}
116160
};
117161
}

‎tools/benchpress/src/console_reporter.es6‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,16 @@ var HEADER_SEPARATORS = ['----', '----', '----', '----', '----', '----', '----']
55
var FOOTER_SEPARATORS = ['====', '====', '====', '====', '====', '====', '===='];
66

77
class ConsoleReporter {
8-
constructor(runId, config) {
8+
constructor(config) {
99
this.config = config;
10-
this.runId = runId;
1110
this.rowFormat = ['%12s'].concat(config.metrics.map(function() {
1211
return '%12s';
1312
})).join(' | ');
1413
}
1514
begin() {
1615
printHeading('BENCHMARK '+this.config.id);
1716
console.log('sample size', this.config.sampleSize);
18-
console.log('run id', this.runId);
17+
console.log('run id', this.config.runId);
1918
console.log('params', JSON.stringify(this.config.params, null, ' '));
2019
printTableHeader(this.rowFormat, ['index', 'forceGc'].concat(this.config.metrics));
2120
}

0 commit comments

Comments
 (0)