Skip to content

Commit 64ef104

Browse files
authored
Add scaling test and better run-clara memory options (#920)
* better jvm memory opts, cleanup * better memory setting * more memory options * add missing help printout * add scaling test * better default * test * bugfix * bugfix * restore
1 parent 88cbd8f commit 64ef104

3 files changed

Lines changed: 122 additions & 8 deletions

File tree

.github/workflows/ci.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ jobs:
199199
- name: run test
200200
run: |
201201
ls -lhtr
202-
./bin/run-clara -y ./etc/services/rgd-clarode.yml -t 4 -n 500 -c ./clara -o ./tmp ./clas_018779.evio.00001
202+
./bin/run-clara -y ./etc/services/rgd-clarode.yml -t 4 -n 500 -c ./clara -o ./tmp ./clas_018779.evio.00001
203203
ls -lhtr
204204
205205
test_coatjava:

bin/run-clara

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ Options:\n
1414
\t-t number of threads (default=2)\n
1515
\t-n number of events (default=-1)\n
1616
\t-m merge output files (see dependencies below)\n
17+
\t-l larger JVM memory requests\n
1718
\t-h print this help and exit\n\n
1819
Merging outputs (-m) requires hipo-utils and yq (https://github.com/mikefarah/yq).'
1920

@@ -25,7 +26,7 @@ function error() {
2526
threads=2
2627
prefix=rec_
2728
CLARA_USER_DATA=.
28-
while getopts y:o:p:c:t:n:qmh opt
29+
while getopts y:o:p:c:t:n:lqmh opt
2930
do
3031
case $opt in
3132
y) yaml=$OPTARG ;;
@@ -34,7 +35,7 @@ do
3435
c) CLARA_HOME=$OPTARG ;;
3536
t) threads=$OPTARG && echo $threads | grep -q -E '^[0-9]+$' || error "-t must be an integer, threads" ;;
3637
n) nevents="-e $OPTARG" && echo "$nevents" | grep -q -E '^-e [0-9]+$' || error "-n must be an integer, events" ;;
37-
g) memory_gb="-e $OPTARG" && echo "$memory_gb" | grep -q -E '^-e [0-9]+$' || error "-g must be an integer, GB of memory" ;;
38+
l) large=1 ;;
3839
m) merge=1 ;;
3940
q) quiet=1 ;;
4041
h) echo -e "\n$usage" && echo -e $info && exit 0 ;;
@@ -43,8 +44,16 @@ done
4344
shift $((OPTIND-1))
4445
inputs=$@
4546

46-
# Configure JVM -Xmx memory setting:
47-
[ -z ${memory_gb+x} ] && memory_gb=$((threads+2))
47+
# Configure JVM memory settings (overridedable via $JAVA_OPTS):
48+
if [ -z ${large+x} ]
49+
then
50+
gb_max=$((threads<3?threads+2:threads+2-threads/4))
51+
gb_init=$((threads<3?threads:threads-threads/3))
52+
else
53+
gb_max=$((threads<3?threads+2:threads+2-threads/8))
54+
gb_init=$((threads<3?threads:threads-threads/6))
55+
fi
56+
java_opts="-Xms${gb_init}g -Xmx${gb_max}g"
4857

4958
# Check configuration:
5059
[ $# -lt 1 ] && error "Input data files are required"
@@ -76,11 +85,14 @@ done
7685
[ $(cat $CLARA_USER_DATA/filelist.txt | wc -l) -gt 0 ] || error "Found no input files"
7786

7887
# Set some JVM options:
79-
export JAVA_OPTS="$JAVA_OPTS -Xmx${memory_gb}g -XX:+IgnoreUnrecognizedVMOptions"
80-
export JAVA_OPTS="$JAVA_OPTS -Djava.io.tmpdir=$CLARA_USER_DATA -Dorg.sqlite.tmpdir=$CLARA_USER_DATA"
88+
JAVA_OPTS="$java_opts $JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions"
89+
JAVA_OPTS="$JAVA_OPTS -Djava.io.tmpdir=$CLARA_USER_DATA -Dorg.sqlite.tmpdir=$CLARA_USER_DATA"
90+
8191
# Set verbosity:
8292
[ -z ${quiet+x} ] && stub=fine || stub=info
83-
export JAVA_OPTS="$JAVA_OPTS -Djava.util.logging.config.file=$CLAS12DIR/etc/logging/$stub.properties"
93+
JAVA_OPTS="$JAVA_OPTS -Djava.util.logging.config.file=$CLAS12DIR/etc/logging/$stub.properties"
94+
95+
export JAVA_OPTS
8496

8597
function get_host_ip() {
8698
if command -v ip >/dev/null 2>&1

libexec/scaling

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
#!/usr/bin/env python3
2+
3+
def cli():
4+
import os,argparse
5+
cli = argparse.ArgumentParser(description='CLARA scaling test')
6+
cli.add_argument('-y',help='YAML file',required=True)
7+
cli.add_argument('-c',help='CLARA_HOME path',default=os.getenv('CLARA_HOME',None))
8+
cli.add_argument('-t',help='threads',default=[2,4],type=int,action='append')
9+
cli.add_argument('-e',help='events per thread',default=100,type=int)
10+
cli.add_argument('input',help='input data file')
11+
cfg = cli.parse_args()
12+
import sys
13+
if cfg.c is None: sys.exit('-c or $CLARA_HOME is required')
14+
return cfg
15+
16+
def run(cmd):
17+
import subprocess
18+
print('scaling >>> '+' '.join(cmd))
19+
p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,universal_newlines=True,encoding='latin-1')
20+
for line in iter(p.stdout.readline, ''):
21+
line = line.strip()
22+
if len(line) > 0:
23+
yield line
24+
p.wait()
25+
if p.returncode != 0:
26+
pass
27+
28+
def benchmark(cfg, threads):
29+
import collections
30+
exiting,benchmarks = False,collections.OrderedDict()
31+
cmd = ['run-clara',
32+
'-c',cfg.c,
33+
'-n',str(cfg.e*threads),
34+
'-t',str(threads),
35+
'-y',cfg.y,
36+
'-o',f'tmp-scaling-{threads}',
37+
cfg.input]
38+
for line in run(cmd):
39+
cols = line.split()
40+
print(line)
41+
try:
42+
if line.find('Benchmark results:') >= 0:
43+
exiting = True
44+
elif line.find('Processing is complete') >= 0:
45+
exiting = False
46+
elif len(cols) > 20:
47+
if line.find('Processed') >= 0:
48+
benchmarks['event'] = float(cols[12])
49+
elif exiting:
50+
# catch-all for services:
51+
if len(cols) > 14:
52+
if 'services' not in benchmarks:
53+
benchmarks['services'] = collections.OrderedDict()
54+
benchmarks['services'][cols[2]] = float(cols[14])
55+
# FIXME: what are these, why don't they add up?
56+
elif line.find('Average processing time') >= 0:
57+
benchmarks['avg'] = float(cols[6])
58+
elif line.find('Total processing time') >= 0:
59+
benchmarks['total'] = float(cols[6])
60+
elif line.find('Total orchestrator time') >= 0:
61+
benchmarks['orch'] = float(cols[6])
62+
except ValueError:
63+
pass
64+
return benchmarks
65+
66+
def table(benchmarks):
67+
table = []
68+
header = [ 'threads' ]
69+
b = benchmarks[0][1]
70+
header.extend([x for x in b if x != 'services'])
71+
if 'services' in b:
72+
header.extend(b['services'].keys())
73+
table.append(header)
74+
for b in benchmarks:
75+
threads,benchmark = b[0],b[1]
76+
row = [threads]
77+
for k in ['event','avg','total','orch','services']:
78+
if k in benchmark:
79+
if k == 'services':
80+
row.extend(benchmark[k].values())
81+
else:
82+
row.append(benchmark[k])
83+
table.append(row)
84+
return table
85+
86+
def show(benchmarks):
87+
for row in table(benchmarks):
88+
print(' '.join([str(x) for x in row]))
89+
90+
def save(benchmarks):
91+
with open('scaling.txt','w') as f:
92+
for row in table(benchmarks):
93+
f.write(' '.join([str(x) for x in row]))
94+
95+
if __name__ == '__main__':
96+
cfg = cli()
97+
benchmarks = []
98+
for threads in cfg.t:
99+
benchmarks.append([threads, benchmark(cfg, threads)])
100+
show(benchmarks)
101+
save(benchmarks)
102+

0 commit comments

Comments
 (0)