-
Notifications
You must be signed in to change notification settings - Fork 0
/
run_all_benches.sh
executable file
·194 lines (180 loc) · 8.42 KB
/
run_all_benches.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
#!/bin/bash
# This script will run caliper benchmarks to compare leveldb as a HLF statedb with SPECIFIED statedb.
# Benchamrks that will be run by INDEX:
# 0 - empty-contract-1of.yaml
# 1 - get-asset.yaml
# 2 - create-asset.yaml
# 3 - delete-asset.yaml //was deleted because there is no method in golang-cheincode. then was returned to the benchmark list.
# 4 - mixed-range-query-pagination.yaml
# 5 - get-asset-batch.yaml
# 6 - create-asset-batch.yaml
#
# ------------------------------------------------------
# Algorithm:
# ------------------------------------------------------
# 1.* (can be ommited initially) Check that desired containers with set database exist. Check that they are not duplicated. if no: do not continue.
#
# 2. Create a catalog for HTML results (the name must be unique)
# 3. Start a loop for INDEX:
# 3.1 do for SPECIFIED statedb
# check that fabric is not run
# 3.1.1 run HLF with SPECIFIED statedb from docker images (start.sh)
# 3.1.2 run INDEXed caliper benchmark saving all cmd output into a file!!!!!
# 3.1.3 check that "report.html" exist, if not - TODO: decide. stop script?
# 3.1.4 put "report.html" to created catalog on step 2
# 3.1.5 rename "report.html" to "report_benchmarkName_SPECIFIEDdb.html"
# 3.1.6 stop HLF
# 3.2 then straightaway do the same for leveldb statedb:
# 3.2.1 run HLF with leveldb statedb from docker images (start.sh)
# 3.2.2 run INDEXed caliper benchmark saving all cmd output into a file!!!!!
# 3.2.3 check that "report.html" exist, if not - TODO: decide. stop script?
# 3.2.4 put "report.html" to created catalog on step 2
# 3.2.5 rename "report.html" to "report_benchmarkName_leveldb.html"
# 3.2.6 stop HLF
#
# ------------------------------------------------------
# Code:
# ------------------------------------------------------
CALIPER_BENCHMARKS_TO_RUN="empty-contract-1of get-asset create-asset create-asset-batch mixed-range-query-pagination get-asset-batch delete-asset"
SupportedDBs="rocksdb boltdb badgerdb (all compares to leveldb)"
LEVELDB="leveldb"
## Parse mode
if [[ $# -lt 1 ]] ; then
echo "Usage (lowcase): ./start.sh statedbname"
exit 0
fi
# parse a statedb subcommand if used
if [[ $# -ge 1 ]] ; then
key="$1"
if [[ "$key" == "leveldb" ]]; then
export STATEDBNAME=leveldb
shift
elif [[ "$key" == "rocksdb" ]]; then
export STATEDBNAME=rocksdb
shift
elif [[ "$key" == "boltdb" ]]; then
export STATEDBNAME=boltdb
shift
elif [[ "$key" == "badgerdb" ]]; then
export STATEDBNAME=badgerdb
shift
else
echo "not existing statedbname, use from the list: ${SupportedDBs}"
exit 0
fi
fi
DIRNAME="bench_${STATEDBNAME}_"`date +"%Y%m%d_%H%M%S"`
mkdir $DIRNAME
#some checks
# check that fabric is not run
if [[ $(docker ps | grep hyperledger) ]]; then
echo "alive fabric containers detected. Please stop them to start this batch of caliper benchmarks."
exit 1
fi
# check that prometheus is run
if ! [[ $(docker ps | grep prometheus) ]]; then
echo "prometheus container wasn't detected. Please run it to start this batch of caliper benchmarks."
exit 1
fi
#checks that images with fabric-leveldb and fabric-STATEDBNAME exist
#TODO this
#check that STATEDBNAME images exist
if ! [[ $(docker images | grep 2.4.0-${STATEDBNAME}) ]]; then
echo "no fabric-${STATEDBNAME} images detected. Please assemble fabric with ${STATEDBNAME} as a state database and make sure that images named with format 2.4.0-${STATEDBNAME}"
exit 1
fi
#check that leveldb images exist
if ! [[ $(docker images | grep 2.4.0-leveldb) ]]; then
echo "no fabric-${LEVELDB} images detected. Please assemble fabric with ${LEVELDB} as a state database and make sure that images named with format 2.4.0-${LEVELDB}"
exit 1
fi
# 3. Start a loop for INDEX:
# 3.1 do for SPECIFIED statedb
for BENCH in ${CALIPER_BENCHMARKS_TO_RUN}
do
# FOR STATEDBNAME:
echo "Running fabric network..."
START_NETWORK_TIME=$(date '+%H%M%S')
set -x
# 3.1.1 run HLF with SPECIFIED statedb from docker images (start.sh)
./start.sh ${STATEDBNAME} >& "./$DIRNAME/${BENCH}_${STATEDBNAME}_output_${START_NETWORK_TIME}.txt"
# 3.1.2 run INDEXed caliper benchmark saving all cmd output into a file!!!!!
{ set +x; } 2>/dev/null
echo ""
echo "===================================="
echo "===================================="
echo "Running $BENCH.yaml..."
echo "===================================="
echo "===================================="
echo ""
START_BENCH_TIME=$(date '+%H:%M:%S')
echo "Benchamrk started at ${START_BENCH_TIME}:" >> "./$DIRNAME/${BENCH}_${STATEDBNAME}_output_${START_NETWORK_TIME}.txt"
set -x
npx caliper launch manager --caliper-workspace . \
--caliper-networkconfig ./fabric-api-solo-node.yaml \
--caliper-benchconfig ./benchmarks/$BENCH.yaml \
--caliper-flow-only-test --caliper-fabric-gateway-enabled >> "./$DIRNAME/${BENCH}_${STATEDBNAME}_output_${START_NETWORK_TIME}.txt"
{ set +x; } 2>/dev/null
# 3.1.3 check that "report.html" exist, if not - echo & continue loop
if ! [[ $(ls | grep report.html) ]]; then
echo ""
echo "WARNING! report.html was not created after benchmark ${BENCH}.yaml"
echo ""
continue
fi
# 3.1.4 put "report.html" to created catalog on step 2
# 3.1.5 rename "report.html" to "report_benchmarkName_SPECIFIEDdb.html"
FINISH_BENCH_TIME=$(date '+%H%M%S')
cp report.html "./$DIRNAME/report_${BENCH}_${STATEDBNAME}_${FINISH_BENCH_TIME}.html"
rm report.html
# 3.1.6 stop HLF
echo "Stopping fabric network..."
./end.sh ${STATEDBNAME} >> "./$DIRNAME/${BENCH}_${STATEDBNAME}_output_${START_NETWORK_TIME}.txt"
#==============================================================================================
#==============================================================================================
#||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
#==============================================================================================
#==============================================================================================
#||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
#==============================================================================================
#==============================================================================================
# 3.2 then straightaway do the same for leveldb statedb:
#FOR LEVELDB DO THE SAME:
echo "Running fabric network..."
START_NETWORK_TIME=$(date '+%H%M%S')
set -x
# 3.2.1 run HLF with leveldb statedb from docker images (start.sh)
./start.sh ${LEVELDB} >& "./$DIRNAME/${BENCH}_${LEVELDB}_output_${START_NETWORK_TIME}.txt"
# 3.2.2 run INDEXed caliper benchmark saving all cmd output into a file!!!!!
{ set +x; } 2>/dev/null
echo ""
echo "===================================="
echo "===================================="
echo "Running $BENCH.yaml..."
echo "===================================="
echo "===================================="
echo ""
START_BENCH_TIME=$(date '+%H:%M:%S')
echo "Benchamrk started at ${START_BENCH_TIME}:" >> "./$DIRNAME/${BENCH}_${LEVELDB}_output_${START_NETWORK_TIME}.txt"
set -x
npx caliper launch manager --caliper-workspace . \
--caliper-networkconfig ./fabric-api-solo-node.yaml \
--caliper-benchconfig ./benchmarks/$BENCH.yaml \
--caliper-flow-only-test --caliper-fabric-gateway-enabled >> "./$DIRNAME/${BENCH}_${LEVELDB}_output_${START_NETWORK_TIME}.txt"
{ set +x; } 2>/dev/null
# 3.2.3 check that "report.html" exist, if not - TODO: decide. stop script?
if ! [[ $(ls | grep report.html) ]]; then
echo ""
echo "WARNING! report.html was not created after benchmark ${BENCH}.yaml"
echo ""
continue
fi
# 3.2.4 put "report.html" to created catalog on step 2
# 3.2.5 rename "report.html" to "report_benchmarkName_leveldb.html"
FINISH_BENCH_TIME=$(date '+%H%M%S')
cp report.html "./$DIRNAME/report_${BENCH}_${LEVELDB}_${FINISH_BENCH_TIME}.html"
rm report.html
# 3.2.6 stop HLF
echo "Stopping fabric network..."
./end.sh ${LEVELDB} >> "./$DIRNAME/${BENCH}_${LEVELDB}_output_${START_NETWORK_TIME}.txt"
done