Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Calculate confidence interval for benchmark measurements

Browse Source
This commit is contained in:
Siarhei Fedartsou 2024-06-15 20:54:47 +02:00
parent 3a5f879749
commit 5585ccc224
2 changed files with 25 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/ci/e2e_benchmark.py
View File

@ -101,8 +101,9 @@ def main():
for _ in range(args.iterations):
random.seed(42)
times = runner.run(args.method, args.host, args.num_requests)
all_times.extend(times)
all_times.append(times)
all_times = np.asarray(all_times)
total_time, total_ci = calculate_confidence_interval(np.sum(all_times, axis=0))
min_time, min_ci = calculate_confidence_interval(np.min(all_times, axis=0))
mean_time, mean_ci = calculate_confidence_interval(np.mean(all_times, axis=0))

44
scripts/ci/run_benchmarks.sh
View File

@ -53,22 +53,22 @@ function run_benchmarks_for_folder {
mkdir -p $RESULTS_FOLDER
BENCHMARKS_FOLDER="$BINARIES_FOLDER/src/benchmarks"
echo "Running match-bench MLD"
$BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
echo "Running match-bench CH"
$BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
echo "Running route-bench MLD"
$BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
echo "Running route-bench CH"
$BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
echo "Running alias"
$BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
echo "Running json-render-bench"
$BENCHMARKS_FOLDER/json-render-bench "$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
echo "Running packedvector-bench"
$BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
echo "Running rtree-bench"
$BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
# echo "Running match-bench MLD"
# $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
# echo "Running match-bench CH"
# $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
# echo "Running route-bench MLD"
# $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
# echo "Running route-bench CH"
# $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
# echo "Running alias"
# $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
# echo "Running json-render-bench"
# $BENCHMARKS_FOLDER/json-render-bench "$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
# echo "Running packedvector-bench"
# $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
# echo "Running rtree-bench"
# $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
cp -rf $OSM_PBF $FOLDER/data.osm.pbf
@ -81,12 +81,12 @@ function run_benchmarks_for_folder {
# echo "Running osrm-contract"
measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench"
for BENCH in nearest table trip route match; do
echo "Running random $BENCH MLD"
$BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" mld $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_mld.bench" || true
echo "Running random $BENCH CH"
$BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" ch $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_ch.bench" || true
done
# for BENCH in nearest table trip route match; do
# echo "Running random $BENCH MLD"
# $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" mld $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_mld.bench" || true
# echo "Running random $BENCH CH"
# $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" ch $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_ch.bench" || true
# done
for ALGORITHM in ch mld; do