Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

wip

Browse Source
This commit is contained in:
Siarhei Fedartsou 2024-06-16 21:42:29 +02:00
parent c6193a9d79
commit 930329b7aa
3 changed files with 77 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/ci/e2e_benchmark.py
View File

@ -130,11 +130,8 @@ def main():
assert all_times.shape == (args.iterations, all_times.shape[1]) assert all_times.shape == (args.iterations, all_times.shape[1])
print('Shape: ', all_times.shape)
total_time, total_ci, total_best = calculate_confidence_interval(np.sum(all_times, axis=1)) total_time, total_ci, total_best = calculate_confidence_interval(np.sum(all_times, axis=1))
print('Ops: ', float(all_times.shape[1]) / np.sum(all_times * 1000, axis=1))
ops_per_sec, ops_per_sec_ci, ops_per_sec_best = calculate_confidence_interval(float(all_times.shape[1]) / np.sum(all_times / 1000, axis=1)) ops_per_sec, ops_per_sec_ci, ops_per_sec_best = calculate_confidence_interval(float(all_times.shape[1]) / np.sum(all_times / 1000, axis=1))
min_time, min_ci, _ = calculate_confidence_interval(np.min(all_times, axis=1)) min_time, min_ci, _ = calculate_confidence_interval(np.min(all_times, axis=1))
mean_time, mean_ci, _ = calculate_confidence_interval(np.mean(all_times, axis=1)) mean_time, mean_ci, _ = calculate_confidence_interval(np.mean(all_times, axis=1))
@ -143,8 +140,8 @@ def main():
perc_99_time, perc_99_ci, _ = calculate_confidence_interval(np.percentile(all_times, 99, axis=1)) perc_99_time, perc_99_ci, _ = calculate_confidence_interval(np.percentile(all_times, 99, axis=1))
max_time, max_ci, _ = calculate_confidence_interval(np.max(all_times, axis=1)) max_time, max_ci, _ = calculate_confidence_interval(np.max(all_times, axis=1))
print(f'Total: {total_time:.2f}ms ± {total_ci:.2f}ms. Best: {total_best:.2f}ms')
print(f'Ops: {ops_per_sec:.2f} ± {ops_per_sec_ci:.2f} ops/s. Best: {ops_per_sec_best:.2f} ops/s') print(f'Ops: {ops_per_sec:.2f} ± {ops_per_sec_ci:.2f} ops/s. Best: {ops_per_sec_best:.2f} ops/s')
print(f'Total: {total_time:.2f}ms ± {total_ci:.2f}ms. Best: {total_best:.2f}ms')
print(f"Min time: {min_time:.2f}ms ± {min_ci:.2f}ms") print(f"Min time: {min_time:.2f}ms ± {min_ci:.2f}ms")
print(f"Mean time: {mean_time:.2f}ms ± {mean_ci:.2f}ms") print(f"Mean time: {mean_time:.2f}ms ± {mean_ci:.2f}ms")
print(f"Median time: {median_time:.2f}ms ± {median_ci:.2f}ms") print(f"Median time: {median_time:.2f}ms ± {median_ci:.2f}ms")

44
scripts/ci/run_benchmarks.sh
View File

@ -53,22 +53,22 @@ function run_benchmarks_for_folder {
mkdir -p $RESULTS_FOLDER mkdir -p $RESULTS_FOLDER
BENCHMARKS_FOLDER="$BINARIES_FOLDER/src/benchmarks" BENCHMARKS_FOLDER="$BINARIES_FOLDER/src/benchmarks"
# echo "Running match-bench MLD" echo "Running match-bench MLD"
# $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench" $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
# echo "Running match-bench CH" echo "Running match-bench CH"
# $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench" $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
# echo "Running route-bench MLD" echo "Running route-bench MLD"
# $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench" $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
# echo "Running route-bench CH" echo "Running route-bench CH"
# $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench" $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
# echo "Running alias" echo "Running alias"
# $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench" $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
# echo "Running json-render-bench" echo "Running json-render-bench"
# $BENCHMARKS_FOLDER/json-render-bench "$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench" $BENCHMARKS_FOLDER/json-render-bench "$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
# echo "Running packedvector-bench" echo "Running packedvector-bench"
# $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench" $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
# echo "Running rtree-bench" echo "Running rtree-bench"
# $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench" $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
cp -rf $OSM_PBF $FOLDER/data.osm.pbf cp -rf $OSM_PBF $FOLDER/data.osm.pbf
@ -81,12 +81,12 @@ function run_benchmarks_for_folder {
# echo "Running osrm-contract" # echo "Running osrm-contract"
measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench" measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench"
# for BENCH in nearest table trip route match; do for BENCH in nearest table trip route match; do
# echo "Running random $BENCH MLD" echo "Running random $BENCH MLD"
# $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" mld $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_mld.bench" || true $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" mld $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_mld.bench" 5 || true
# echo "Running random $BENCH CH" echo "Running random $BENCH CH"
# $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" ch $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_ch.bench" || true $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" ch $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_ch.bench" 5 || true
# done done
for ALGORITHM in ch mld; do for ALGORITHM in ch mld; do

127
src/benchmarks/bench.cpp
View File

@ -45,8 +45,12 @@ class GPSTraces
std::vector<osrm::util::Coordinate> coordinates; std::vector<osrm::util::Coordinate> coordinates;
mutable std::mt19937 gen; mutable std::mt19937 gen;
int seed;
public: public:
GPSTraces(int seed) : gen(std::random_device{}()) { gen.seed(seed); } GPSTraces(int seed) : gen(std::random_device{}()), seed(seed) { gen.seed(seed); }
void resetSeed() const { gen.seed(seed); }
bool readCSV(const std::string &filename) bool readCSV(const std::string &filename)
{ {
@ -113,7 +117,7 @@ class GPSTraces
class Statistics class Statistics
{ {
public: public:
void push(double timeMs) void push(double timeMs, int /*iteration*/ = 0)
{ {
times.push_back(timeMs); times.push_back(timeMs);
sorted = false; sorted = false;
@ -168,7 +172,7 @@ std::ostream &operator<<(std::ostream &os, Statistics &statistics)
return os; return os;
} }
void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations)
{ {
struct Benchmark struct Benchmark
{ {
@ -184,43 +188,49 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
{ {
Statistics statistics; Statistics statistics;
auto NUM = 10000; for (int iteration = 0; iteration < iterations; ++iteration)
for (int i = 0; i < NUM; ++i)
{ {
RouteParameters params; gpsTraces.resetSeed();
params.overview = benchmark.overview;
params.steps = benchmark.steps;
for (size_t i = 0; i < benchmark.coordinates; ++i) auto NUM = 1000;
for (int i = 0; i < NUM; ++i)
{ {
params.coordinates.push_back(gpsTraces.getRandomCoordinate()); RouteParameters params;
} params.overview = benchmark.overview;
params.steps = benchmark.steps;
if (benchmark.alternatives) for (size_t i = 0; i < benchmark.coordinates; ++i)
{
params.alternatives = *benchmark.alternatives;
}
if (benchmark.radius)
{
params.radiuses = std::vector<boost::optional<double>>(
params.coordinates.size(), boost::make_optional(*benchmark.radius));
}
engine::api::ResultT result = json::Object();
TIMER_START(routes);
const auto rc = osrm.Route(params, result);
TIMER_STOP(routes);
statistics.push(TIMER_MSEC(routes));
auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok || json_result.values.find("routes") == json_result.values.end())
{
auto code = std::get<json::String>(json_result.values["code"]).value;
if (code != "NoSegment" && code != "NoRoute")
{ {
throw std::runtime_error{"Couldn't route: " + code}; params.coordinates.push_back(gpsTraces.getRandomCoordinate());
}
if (benchmark.alternatives)
{
params.alternatives = *benchmark.alternatives;
}
if (benchmark.radius)
{
params.radiuses = std::vector<boost::optional<double>>(
params.coordinates.size(), boost::make_optional(*benchmark.radius));
}
engine::api::ResultT result = json::Object();
TIMER_START(routes);
const auto rc = osrm.Route(params, result);
TIMER_STOP(routes);
statistics.push(TIMER_MSEC(routes), iteration);
auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok ||
json_result.values.find("routes") == json_result.values.end())
{
auto code = std::get<json::String>(json_result.values["code"]).value;
if (code != "NoSegment" && code != "NoRoute")
{
throw std::runtime_error{"Couldn't route: " + code};
}
} }
} }
} }
@ -234,11 +244,6 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
RouteParameters::OverviewType::Full, RouteParameters::OverviewType::Full,
true, true,
std::nullopt}, std::nullopt},
{"10000 routes, 2 coordinates, no alternatives, overview=full, steps=true",
2,
RouteParameters::OverviewType::Full,
true,
std::nullopt},
{"10000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true", {"10000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true",
2, 2,
RouteParameters::OverviewType::Full, RouteParameters::OverviewType::Full,
@ -249,36 +254,11 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
RouteParameters::OverviewType::False, RouteParameters::OverviewType::False,
false, false,
std::nullopt}, std::nullopt},
{"10000 routes, 2 coordinates, no alternatives, overview=false, steps=false",
2,
RouteParameters::OverviewType::False,
false,
std::nullopt},
{"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false", {"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false",
2, 2,
RouteParameters::OverviewType::False, RouteParameters::OverviewType::False,
false, false,
3}, 3}};
{"10000 routes, 3 coordinates, no alternatives, overview=false, steps=false, radius=750",
3,
RouteParameters::OverviewType::False,
false,
std::nullopt,
750},
{"10000 routes, 2 coordinates, no alternatives, overview=false, steps=false, radius=750",
2,
RouteParameters::OverviewType::False,
false,
std::nullopt,
750},
{"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false, radius=750",
2,
RouteParameters::OverviewType::False,
false,
3,
750}
};
for (const auto &benchmark : benchmarks) for (const auto &benchmark : benchmarks)
{ {
@ -502,8 +482,7 @@ void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
std::vector<Benchmark> benchmarks = {{"250 tables, 3 coordinates", 3}, std::vector<Benchmark> benchmarks = {{"250 tables, 3 coordinates", 3},
{"250 tables, 25 coordinates", 25}, {"250 tables, 25 coordinates", 25},
{"250 tables, 50 coordinates", 50}, {"250 tables, 50 coordinates", 50}};
{"250 tables, 100 coordinates", 100}};
for (const auto &benchmark : benchmarks) for (const auto &benchmark : benchmarks)
{ {
@ -516,11 +495,11 @@ void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
int main(int argc, const char *argv[]) int main(int argc, const char *argv[])
try try
{ {
if (argc < 5) if (argc < 6)
{ {
std::cerr std::cerr << "Usage: " << argv[0]
<< "Usage: " << argv[0] << " data.osrm <mld|ch> <path to GPS traces.csv> "
<< " data.osrm <mld|ch> <path to GPS traces.csv> <route|match|trip|table|nearest>\n"; "<route|match|trip|table|nearest> <number_of_iterations>\n";
return EXIT_FAILURE; return EXIT_FAILURE;
} }
@ -537,11 +516,13 @@ try
GPSTraces gpsTraces{42}; GPSTraces gpsTraces{42};
gpsTraces.readCSV(argv[3]); gpsTraces.readCSV(argv[3]);
int iterations = std::stoi(argv[5]);
const auto benchmarkToRun = std::string{argv[4]}; const auto benchmarkToRun = std::string{argv[4]};
if (benchmarkToRun == "route") if (benchmarkToRun == "route")
{ {
runRouteBenchmark(osrm, gpsTraces); runRouteBenchmark(osrm, gpsTraces, iterations);
} }
else if (benchmarkToRun == "match") else if (benchmarkToRun == "match")
{ {