diff --git a/scripts/ci/e2e_benchmark.py b/scripts/ci/e2e_benchmark.py index 7a5fe2726..8e2e51944 100644 --- a/scripts/ci/e2e_benchmark.py +++ b/scripts/ci/e2e_benchmark.py @@ -130,11 +130,8 @@ def main(): assert all_times.shape == (args.iterations, all_times.shape[1]) - print('Shape: ', all_times.shape) total_time, total_ci, total_best = calculate_confidence_interval(np.sum(all_times, axis=1)) - print('Ops: ', float(all_times.shape[1]) / np.sum(all_times * 1000, axis=1)) - ops_per_sec, ops_per_sec_ci, ops_per_sec_best = calculate_confidence_interval(float(all_times.shape[1]) / np.sum(all_times / 1000, axis=1)) min_time, min_ci, _ = calculate_confidence_interval(np.min(all_times, axis=1)) mean_time, mean_ci, _ = calculate_confidence_interval(np.mean(all_times, axis=1)) @@ -143,8 +140,8 @@ def main(): perc_99_time, perc_99_ci, _ = calculate_confidence_interval(np.percentile(all_times, 99, axis=1)) max_time, max_ci, _ = calculate_confidence_interval(np.max(all_times, axis=1)) - print(f'Total: {total_time:.2f}ms ± {total_ci:.2f}ms. Best: {total_best:.2f}ms') print(f'Ops: {ops_per_sec:.2f} ± {ops_per_sec_ci:.2f} ops/s. Best: {ops_per_sec_best:.2f} ops/s') + print(f'Total: {total_time:.2f}ms ± {total_ci:.2f}ms. Best: {total_best:.2f}ms') print(f"Min time: {min_time:.2f}ms ± {min_ci:.2f}ms") print(f"Mean time: {mean_time:.2f}ms ± {mean_ci:.2f}ms") print(f"Median time: {median_time:.2f}ms ± {median_ci:.2f}ms") diff --git a/scripts/ci/run_benchmarks.sh b/scripts/ci/run_benchmarks.sh index f117f95a3..2e01b5ce5 100755 --- a/scripts/ci/run_benchmarks.sh +++ b/scripts/ci/run_benchmarks.sh @@ -53,22 +53,22 @@ function run_benchmarks_for_folder { mkdir -p $RESULTS_FOLDER BENCHMARKS_FOLDER="$BINARIES_FOLDER/src/benchmarks" - # echo "Running match-bench MLD" - # $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench" - # echo "Running match-bench CH" - # $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench" - # echo "Running route-bench MLD" - # $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench" - # echo "Running route-bench CH" - # $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench" - # echo "Running alias" - # $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench" - # echo "Running json-render-bench" - # $BENCHMARKS_FOLDER/json-render-bench "$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench" - # echo "Running packedvector-bench" - # $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench" - # echo "Running rtree-bench" - # $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench" + echo "Running match-bench MLD" + $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench" + echo "Running match-bench CH" + $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench" + echo "Running route-bench MLD" + $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench" + echo "Running route-bench CH" + $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench" + echo "Running alias" + $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench" + echo "Running json-render-bench" + $BENCHMARKS_FOLDER/json-render-bench "$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench" + echo "Running packedvector-bench" + $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench" + echo "Running rtree-bench" + $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench" cp -rf $OSM_PBF $FOLDER/data.osm.pbf @@ -81,12 +81,12 @@ function run_benchmarks_for_folder { # echo "Running osrm-contract" measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench" - # for BENCH in nearest table trip route match; do - # echo "Running random $BENCH MLD" - # $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" mld $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_mld.bench" || true - # echo "Running random $BENCH CH" - # $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" ch $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_ch.bench" || true - # done + for BENCH in nearest table trip route match; do + echo "Running random $BENCH MLD" + $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" mld $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_mld.bench" 5 || true + echo "Running random $BENCH CH" + $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" ch $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_ch.bench" 5 || true + done for ALGORITHM in ch mld; do diff --git a/src/benchmarks/bench.cpp b/src/benchmarks/bench.cpp index ef53c398d..26e10a361 100644 --- a/src/benchmarks/bench.cpp +++ b/src/benchmarks/bench.cpp @@ -45,8 +45,12 @@ class GPSTraces std::vector coordinates; mutable std::mt19937 gen; + int seed; + public: - GPSTraces(int seed) : gen(std::random_device{}()) { gen.seed(seed); } + GPSTraces(int seed) : gen(std::random_device{}()), seed(seed) { gen.seed(seed); } + + void resetSeed() const { gen.seed(seed); } bool readCSV(const std::string &filename) { @@ -113,7 +117,7 @@ class GPSTraces class Statistics { public: - void push(double timeMs) + void push(double timeMs, int /*iteration*/ = 0) { times.push_back(timeMs); sorted = false; @@ -168,7 +172,7 @@ std::ostream &operator<<(std::ostream &os, Statistics &statistics) return os; } -void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) +void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations) { struct Benchmark { @@ -184,43 +188,49 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) { Statistics statistics; - auto NUM = 10000; - for (int i = 0; i < NUM; ++i) + for (int iteration = 0; iteration < iterations; ++iteration) { - RouteParameters params; - params.overview = benchmark.overview; - params.steps = benchmark.steps; + gpsTraces.resetSeed(); - for (size_t i = 0; i < benchmark.coordinates; ++i) + auto NUM = 1000; + for (int i = 0; i < NUM; ++i) { - params.coordinates.push_back(gpsTraces.getRandomCoordinate()); - } + RouteParameters params; + params.overview = benchmark.overview; + params.steps = benchmark.steps; - if (benchmark.alternatives) - { - params.alternatives = *benchmark.alternatives; - } - - if (benchmark.radius) - { - params.radiuses = std::vector>( - params.coordinates.size(), boost::make_optional(*benchmark.radius)); - } - - engine::api::ResultT result = json::Object(); - TIMER_START(routes); - const auto rc = osrm.Route(params, result); - TIMER_STOP(routes); - - statistics.push(TIMER_MSEC(routes)); - - auto &json_result = std::get(result); - if (rc != Status::Ok || json_result.values.find("routes") == json_result.values.end()) - { - auto code = std::get(json_result.values["code"]).value; - if (code != "NoSegment" && code != "NoRoute") + for (size_t i = 0; i < benchmark.coordinates; ++i) { - throw std::runtime_error{"Couldn't route: " + code}; + params.coordinates.push_back(gpsTraces.getRandomCoordinate()); + } + + if (benchmark.alternatives) + { + params.alternatives = *benchmark.alternatives; + } + + if (benchmark.radius) + { + params.radiuses = std::vector>( + params.coordinates.size(), boost::make_optional(*benchmark.radius)); + } + + engine::api::ResultT result = json::Object(); + TIMER_START(routes); + const auto rc = osrm.Route(params, result); + TIMER_STOP(routes); + + statistics.push(TIMER_MSEC(routes), iteration); + + auto &json_result = std::get(result); + if (rc != Status::Ok || + json_result.values.find("routes") == json_result.values.end()) + { + auto code = std::get(json_result.values["code"]).value; + if (code != "NoSegment" && code != "NoRoute") + { + throw std::runtime_error{"Couldn't route: " + code}; + } } } } @@ -234,11 +244,6 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) RouteParameters::OverviewType::Full, true, std::nullopt}, - {"10000 routes, 2 coordinates, no alternatives, overview=full, steps=true", - 2, - RouteParameters::OverviewType::Full, - true, - std::nullopt}, {"10000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true", 2, RouteParameters::OverviewType::Full, @@ -249,36 +254,11 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) RouteParameters::OverviewType::False, false, std::nullopt}, - {"10000 routes, 2 coordinates, no alternatives, overview=false, steps=false", - 2, - RouteParameters::OverviewType::False, - false, - std::nullopt}, {"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false", 2, RouteParameters::OverviewType::False, false, - 3}, - {"10000 routes, 3 coordinates, no alternatives, overview=false, steps=false, radius=750", - 3, - RouteParameters::OverviewType::False, - false, - std::nullopt, - 750}, - {"10000 routes, 2 coordinates, no alternatives, overview=false, steps=false, radius=750", - 2, - RouteParameters::OverviewType::False, - false, - std::nullopt, - 750}, - {"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false, radius=750", - 2, - RouteParameters::OverviewType::False, - false, - 3, - 750} - - }; + 3}}; for (const auto &benchmark : benchmarks) { @@ -502,8 +482,7 @@ void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) std::vector benchmarks = {{"250 tables, 3 coordinates", 3}, {"250 tables, 25 coordinates", 25}, - {"250 tables, 50 coordinates", 50}, - {"250 tables, 100 coordinates", 100}}; + {"250 tables, 50 coordinates", 50}}; for (const auto &benchmark : benchmarks) { @@ -516,11 +495,11 @@ void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) int main(int argc, const char *argv[]) try { - if (argc < 5) + if (argc < 6) { - std::cerr - << "Usage: " << argv[0] - << " data.osrm \n"; + std::cerr << "Usage: " << argv[0] + << " data.osrm " + " \n"; return EXIT_FAILURE; } @@ -537,11 +516,13 @@ try GPSTraces gpsTraces{42}; gpsTraces.readCSV(argv[3]); + int iterations = std::stoi(argv[5]); + const auto benchmarkToRun = std::string{argv[4]}; if (benchmarkToRun == "route") { - runRouteBenchmark(osrm, gpsTraces); + runRouteBenchmark(osrm, gpsTraces, iterations); } else if (benchmarkToRun == "match") {