wip

scripts/ci/run_benchmarks.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -eou pipefail
 
+export LD_LIBRARY_PATH="/home/sfedartsou/.conan/data/onetbb/2021.12.0/_/_/package/14d80dcc6e4fac53fcd6bcf3899648ac2c72c6aa/lib/"
+
 function usage {
     echo "Usage: $0 -f <folder> -r <results_folder> -s <scripts_folder> -b <binaries_folder> -o <osm_pbf> -g <gps_traces>"
     exit 1
@@ -52,69 +54,75 @@ function measure_peak_ram_and_time {
 function run_benchmarks_for_folder {
     mkdir -p $RESULTS_FOLDER
 
+    rm -rf $RESULTS_FOLDER/*
+
     BENCHMARKS_FOLDER="$BINARIES_FOLDER/src/benchmarks"
     echo "Running match-bench MLD"
-    $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
+    perf stat -r 10 $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld &> "$RESULTS_FOLDER/match_mld.bench"
     echo "Running match-bench CH"
-    $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
+    perf stat -r 10 $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch &> "$RESULTS_FOLDER/match_ch.bench"
-    echo "Running route-bench MLD"
+    echo "Running match-bench MLD"
-    $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
+    perf stat -r 10 $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld &> "$RESULTS_FOLDER/match_mld2.bench"
-    echo "Running route-bench CH"
+    echo "Running match-bench CH"
-    $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
+    perf stat -r 10 $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch &> "$RESULTS_FOLDER/match_ch2.bench"
-    echo "Running alias"
+    # echo "Running route-bench MLD"
-    $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
+    # $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
-    echo "Running json-render-bench"
+    # echo "Running route-bench CH"
-    $BENCHMARKS_FOLDER/json-render-bench  "$FOLDER/test/data/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
+    # $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
-    echo "Running packedvector-bench"
+    # echo "Running alias"
-    $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
+    # $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
-    echo "Running rtree-bench"
+    # echo "Running json-render-bench"
-    $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
+    # $BENCHMARKS_FOLDER/json-render-bench  "$FOLDER/test/data/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
+    # echo "Running packedvector-bench"
+    # $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
+    # echo "Running rtree-bench"
+    # $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
 
-    cp -rf $OSM_PBF $FOLDER/data.osm.pbf
+    # cp -rf $OSM_PBF $FOLDER/data.osm.pbf
 
-    echo "Running osrm-extract"
+    # echo "Running osrm-extract"
-    measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-extract -p $FOLDER/profiles/car.lua $FOLDER/data.osm.pbf" "$RESULTS_FOLDER/osrm_extract.bench"
+    # measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-extract -p $FOLDER/profiles/car.lua $FOLDER/data.osm.pbf" "$RESULTS_FOLDER/osrm_extract.bench"
-    echo "Running osrm-partition"
+    # echo "Running osrm-partition"
-    measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-partition $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_partition.bench"
+    # measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-partition $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_partition.bench"
-    echo "Running osrm-customize"
+    # echo "Running osrm-customize"
-    measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-customize $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_customize.bench"
+    # measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-customize $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_customize.bench"
-    echo "Running osrm-contract"
+    # echo "Running osrm-contract"
-    measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench"
+    # measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench"
 
-    for ALGORITHM in ch mld; do
+    # for ALGORITHM in ch mld; do
-        for BENCH in nearest table trip route match; do
+    #     for BENCH in nearest table trip route match; do
-            echo "Running random $BENCH $ALGORITHM"
+    #         echo "Running random $BENCH $ALGORITHM"
-            START=$(date +%s.%N)
+    #         START=$(date +%s.%N)
-            $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" $ALGORITHM $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_${ALGORITHM}.bench" 5 || true
+    #         $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" $ALGORITHM $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_${ALGORITHM}.bench" 5 || true
-            END=$(date +%s.%N)
+    #         END=$(date +%s.%N)
-            DIFF=$(echo "$END - $START" | bc)
+    #         DIFF=$(echo "$END - $START" | bc)
-            echo "Took: ${DIFF}s"
+    #         echo "Took: ${DIFF}s"
-        done
+    #     done
-    done
+    # done
 
 
-    for ALGORITHM in ch mld; do
+    # for ALGORITHM in ch mld; do
-        $BINARIES_FOLDER/osrm-routed --algorithm $ALGORITHM $FOLDER/data.osrm > /dev/null 2>&1 &
+    #     $BINARIES_FOLDER/osrm-routed --algorithm $ALGORITHM $FOLDER/data.osrm > /dev/null 2>&1 &
-        OSRM_ROUTED_PID=$!
+    #     OSRM_ROUTED_PID=$!
 
-        # wait for osrm-routed to start
+    #     # wait for osrm-routed to start
-        if ! curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" > /dev/null 2>&1; then
+    #     if ! curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" > /dev/null 2>&1; then
-            echo "osrm-routed failed to start for algorithm $ALGORITHM"
+    #         echo "osrm-routed failed to start for algorithm $ALGORITHM"
-            kill -9 $OSRM_ROUTED_PID
+    #         kill -9 $OSRM_ROUTED_PID
-            continue
+    #         continue
-        fi
+    #     fi
 
-        for METHOD in route nearest trip table match; do
+    #     for METHOD in route nearest trip table match; do
-            echo "Running e2e benchmark for $METHOD $ALGORITHM"
+    #         echo "Running e2e benchmark for $METHOD $ALGORITHM"
-            START=$(date +%s.%N)
+    #         START=$(date +%s.%N)
-            python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --iterations 5 --num_requests 1000 --gps_traces_file_path $GPS_TRACES > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench
+    #         python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --iterations 5 --num_requests 1000 --gps_traces_file_path $GPS_TRACES > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench
-            END=$(date +%s.%N)
+    #         END=$(date +%s.%N)
-            DIFF=$(echo "$END - $START" | bc)
+    #         DIFF=$(echo "$END - $START" | bc)
-            echo "Took: ${DIFF}s"
+    #         echo "Took: ${DIFF}s"
-        done
+    #     done
 
-        kill -9 $OSRM_ROUTED_PID
+    #     kill -9 $OSRM_ROUTED_PID
-    done
+    # done
 }
 
 run_benchmarks_for_folder