Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

wip

Browse Source
This commit is contained in:
Siarhei Fedartsou 2024-06-18 18:08:47 +02:00
parent 2d3782344d
commit 7c2a2a2e72
2 changed files with 859 additions and 850 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1227
.github/workflows/osrm-backend.yml vendored
View File

File diff suppressed because it is too large Load Diff

334
src/benchmarks/bench.cpp
View File

@ -37,8 +37,6 @@ using namespace osrm;
namespace namespace
{ {
class GPSTraces class GPSTraces
{ {
private: private:
@ -117,21 +115,27 @@ class GPSTraces
}; };
// Struct to hold confidence interval data // Struct to hold confidence interval data
struct ConfidenceInterval { struct ConfidenceInterval
{
double mean; double mean;
double confidence; double confidence;
double min; double min;
}; };
// Helper function to calculate the bootstrap confidence interval // Helper function to calculate the bootstrap confidence interval
ConfidenceInterval confidenceInterval(const std::vector<double>& data, int num_samples = 1000, double confidence_level = 0.95) { ConfidenceInterval confidenceInterval(const std::vector<double> &data,
int num_samples = 1000,
double confidence_level = 0.95)
{
std::vector<double> means; std::vector<double> means;
std::default_random_engine generator; std::default_random_engine generator;
std::uniform_int_distribution<int> distribution(0, data.size() - 1); std::uniform_int_distribution<int> distribution(0, data.size() - 1);
for (int i = 0; i < num_samples; ++i) { for (int i = 0; i < num_samples; ++i)
{
std::vector<double> sample; std::vector<double> sample;
for (size_t j = 0; j < data.size(); ++j) { for (size_t j = 0; j < data.size(); ++j)
{
sample.push_back(data[distribution(generator)]); sample.push_back(data[distribution(generator)]);
} }
double sample_mean = std::accumulate(sample.begin(), sample.end(), 0.0) / sample.size(); double sample_mean = std::accumulate(sample.begin(), sample.end(), 0.0) / sample.size();
@ -143,55 +147,64 @@ ConfidenceInterval confidenceInterval(const std::vector<double>& data, int num_s
double upper_bound = means[(int)((1 + confidence_level) / 2 * num_samples)]; double upper_bound = means[(int)((1 + confidence_level) / 2 * num_samples)];
double mean = std::accumulate(means.begin(), means.end(), 0.0) / means.size(); double mean = std::accumulate(means.begin(), means.end(), 0.0) / means.size();
ConfidenceInterval ci = {mean, (upper_bound - lower_bound) / 2, *std::min_element(data.begin(), data.end())}; ConfidenceInterval ci = {
mean, (upper_bound - lower_bound) / 2, *std::min_element(data.begin(), data.end())};
return ci; return ci;
} }
class Statistics class Statistics
{ {
public: public:
explicit Statistics(int iterations) : times(iterations) {} explicit Statistics(int iterations) : times(iterations) {}
void push(double timeMs, int iteration) { void push(double timeMs, int iteration) { times[iteration].push_back(timeMs); }
times[iteration].push_back(timeMs);
}
ConfidenceInterval mean() { ConfidenceInterval mean()
{
std::vector<double> means; std::vector<double> means;
for (const auto& iter_times : times) { for (const auto &iter_times : times)
means.push_back(std::accumulate(iter_times.begin(), iter_times.end(), 0.0) / iter_times.size()); {
means.push_back(std::accumulate(iter_times.begin(), iter_times.end(), 0.0) /
iter_times.size());
} }
return confidenceInterval(means); return confidenceInterval(means);
} }
ConfidenceInterval total() { ConfidenceInterval total()
{
std::vector<double> sums; std::vector<double> sums;
for (const auto& iter_times : times) { for (const auto &iter_times : times)
{
sums.push_back(std::accumulate(iter_times.begin(), iter_times.end(), 0.0)); sums.push_back(std::accumulate(iter_times.begin(), iter_times.end(), 0.0));
} }
return confidenceInterval(sums); return confidenceInterval(sums);
} }
ConfidenceInterval min() { ConfidenceInterval min()
{
std::vector<double> mins; std::vector<double> mins;
for (const auto& iter_times : times) { for (const auto &iter_times : times)
{
mins.push_back(*std::min_element(iter_times.begin(), iter_times.end())); mins.push_back(*std::min_element(iter_times.begin(), iter_times.end()));
} }
return confidenceInterval(mins); return confidenceInterval(mins);
} }
ConfidenceInterval max() { ConfidenceInterval max()
{
std::vector<double> maxs; std::vector<double> maxs;
for (const auto& iter_times : times) { for (const auto &iter_times : times)
{
maxs.push_back(*std::max_element(iter_times.begin(), iter_times.end())); maxs.push_back(*std::max_element(iter_times.begin(), iter_times.end()));
} }
return confidenceInterval(maxs); return confidenceInterval(maxs);
} }
ConfidenceInterval percentile(double p) { ConfidenceInterval percentile(double p)
{
std::vector<double> percentiles; std::vector<double> percentiles;
for (const auto& iter_times : times) { for (const auto &iter_times : times)
{
auto sorted_times = iter_times; auto sorted_times = iter_times;
std::sort(sorted_times.begin(), sorted_times.end()); std::sort(sorted_times.begin(), sorted_times.end());
percentiles.push_back(sorted_times[static_cast<size_t>(p * sorted_times.size())]); percentiles.push_back(sorted_times[static_cast<size_t>(p * sorted_times.size())]);
@ -199,20 +212,24 @@ class Statistics
return confidenceInterval(percentiles); return confidenceInterval(percentiles);
} }
ConfidenceInterval ops_per_sec() { ConfidenceInterval ops_per_sec()
{
std::vector<double> ops; std::vector<double> ops;
for (const auto& iter_times : times) { for (const auto &iter_times : times)
{
double total_time = std::accumulate(iter_times.begin(), iter_times.end(), 0.0) / 1000.0; double total_time = std::accumulate(iter_times.begin(), iter_times.end(), 0.0) / 1000.0;
ops.push_back(iter_times.size() / total_time); ops.push_back(iter_times.size() / total_time);
} }
return confidenceInterval(ops); return confidenceInterval(ops);
} }
private: private:
// vector of times for each iteration // vector of times for each iteration
std::vector<std::vector<double>> times; std::vector<std::vector<double>> times;
}; };
std::ostream& operator<<(std::ostream& os, Statistics& statistics) { std::ostream &operator<<(std::ostream &os, Statistics &statistics)
{
os << std::fixed << std::setprecision(2); os << std::fixed << std::setprecision(2);
ConfidenceInterval mean_ci = statistics.mean(); ConfidenceInterval mean_ci = statistics.mean();
@ -222,8 +239,10 @@ std::ostream& operator<<(std::ostream& os, Statistics& statistics) {
ConfidenceInterval p99_ci = statistics.percentile(0.99); ConfidenceInterval p99_ci = statistics.percentile(0.99);
ConfidenceInterval ops_ci = statistics.ops_per_sec(); ConfidenceInterval ops_ci = statistics.ops_per_sec();
os << "ops: " << ops_ci.mean << " ± " << ops_ci.confidence << " ops/s. " << "best: " << ops_ci.min << "ops/s." << std::endl; os << "ops: " << ops_ci.mean << " ± " << ops_ci.confidence << " ops/s. "
os << "total: " << total_ci.mean << " ± " << total_ci.confidence << "ms. " << "best: " << total_ci.min << "ms." << std::endl; << "best: " << ops_ci.min << "ops/s." << std::endl;
os << "total: " << total_ci.mean << " ± " << total_ci.confidence << "ms. "
<< "best: " << total_ci.min << "ms." << std::endl;
os << "avg: " << mean_ci.mean << " ± " << mean_ci.confidence << "ms" << std::endl; os << "avg: " << mean_ci.mean << " ± " << mean_ci.confidence << "ms" << std::endl;
os << "min: " << min_ci.mean << " ± " << min_ci.confidence << "ms" << std::endl; os << "min: " << min_ci.mean << " ± " << min_ci.confidence << "ms" << std::endl;
os << "max: " << max_ci.mean << " ± " << max_ci.confidence << "ms" << std::endl; os << "max: " << max_ci.mean << " ± " << max_ci.confidence << "ms" << std::endl;
@ -232,8 +251,34 @@ std::ostream& operator<<(std::ostream& os, Statistics& statistics) {
return os; return os;
} }
template <typename Benchmark, typename BenchmarkBody>
void runBenchmarks(const std::vector<Benchmark> &benchmarks,
int iterations,
int opsPerIteration,
const OSRM &osrm,
const GPSTraces &gpsTraces,
const BenchmarkBody &benchmarkBody)
{
for (const auto &benchmark : benchmarks)
{
Statistics statistics{iterations};
for (int iteration = 0; iteration < iterations; ++iteration)
{
gpsTraces.resetSeed();
for (int i = 0; i < opsPerIteration; ++i)
{
benchmarkBody(iteration, benchmark, osrm, gpsTraces, statistics);
}
}
std::cout << benchmark.name << std::endl;
std::cout << statistics << std::endl;
}
}
void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations) void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations)
{ {
struct Benchmark struct Benchmark
{ {
std::string name; std::string name;
@ -243,17 +288,38 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterati
std::optional<size_t> alternatives = std::nullopt; std::optional<size_t> alternatives = std::nullopt;
std::optional<double> radius = std::nullopt; std::optional<double> radius = std::nullopt;
}; };
std::vector<Benchmark> benchmarks = {
{"10000 routes, 3 coordinates, no alternatives, overview=full, steps=true",
3,
RouteParameters::OverviewType::Full,
true,
std::nullopt},
{"10000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true",
2,
RouteParameters::OverviewType::Full,
true,
3},
{"10000 routes, 3 coordinates, no alternatives, overview=false, steps=false",
3,
RouteParameters::OverviewType::False,
false,
std::nullopt},
{"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false",
2,
RouteParameters::OverviewType::False,
false,
3}};
auto run_benchmark = [&](const Benchmark &benchmark) runBenchmarks(benchmarks,
{ iterations,
Statistics statistics{iterations}; 10000,
osrm,
for (int iteration = 0; iteration < iterations; ++iteration) gpsTraces,
{ [](int iteration,
gpsTraces.resetSeed(); const Benchmark &benchmark,
const OSRM &osrm,
auto NUM = 1000; const GPSTraces &gpsTraces,
for (int i = 0; i < NUM; ++i) Statistics &statistics)
{ {
RouteParameters params; RouteParameters params;
params.overview = benchmark.overview; params.overview = benchmark.overview;
@ -280,8 +346,6 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterati
const auto rc = osrm.Route(params, result); const auto rc = osrm.Route(params, result);
TIMER_STOP(routes); TIMER_STOP(routes);
statistics.push(TIMER_MSEC(routes), iteration);
auto &json_result = std::get<json::Object>(result); auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok || if (rc != Status::Ok ||
json_result.values.find("routes") == json_result.values.end()) json_result.values.find("routes") == json_result.values.end())
@ -292,38 +356,12 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterati
throw std::runtime_error{"Couldn't route: " + code}; throw std::runtime_error{"Couldn't route: " + code};
} }
} }
} else
}
std::cout << benchmark.name << std::endl;
std::cout << statistics << std::endl;
};
std::vector<Benchmark> benchmarks = {
{"10000 routes, 3 coordinates, no alternatives, overview=full, steps=true",
3,
RouteParameters::OverviewType::Full,
true,
std::nullopt},
{"10000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true",
2,
RouteParameters::OverviewType::Full,
true,
3},
{"10000 routes, 3 coordinates, no alternatives, overview=false, steps=false",
3,
RouteParameters::OverviewType::False,
false,
std::nullopt},
{"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false",
2,
RouteParameters::OverviewType::False,
false,
3}};
for (const auto &benchmark : benchmarks)
{ {
run_benchmark(benchmark);
statistics.push(TIMER_MSEC(routes), iteration);
} }
});
} }
void runMatchBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations) void runMatchBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations)
@ -334,15 +372,20 @@ void runMatchBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterati
std::optional<size_t> radius = std::nullopt; std::optional<size_t> radius = std::nullopt;
}; };
auto run_benchmark = [&](const Benchmark &benchmark) std::vector<Benchmark> benchmarks = {{"1000 matches, default radius"},
{ {"1000 matches, radius=10", 10},
Statistics statistics{iterations}; {"1000 matches, radius=20", 20}};
auto NUM = 1000; runBenchmarks(benchmarks,
for (int iteration = 0; iteration < iterations; ++iteration) iterations,
{ 1000,
gpsTraces.resetSeed(); osrm,
for (int i = 0; i < NUM; ++i) gpsTraces,
[](int iteration,
const Benchmark &benchmark,
const OSRM &osrm,
const GPSTraces &gpsTraces,
Statistics &statistics)
{ {
engine::api::ResultT result = json::Object(); engine::api::ResultT result = json::Object();
@ -361,8 +404,6 @@ void runMatchBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterati
const auto rc = osrm.Match(params, result); const auto rc = osrm.Match(params, result);
TIMER_STOP(match); TIMER_STOP(match);
statistics.push(TIMER_MSEC(match), iteration);
auto &json_result = std::get<json::Object>(result); auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok || if (rc != Status::Ok ||
json_result.values.find("matchings") == json_result.values.end()) json_result.values.find("matchings") == json_result.values.end())
@ -373,21 +414,11 @@ void runMatchBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterati
throw std::runtime_error{"Couldn't route: " + code}; throw std::runtime_error{"Couldn't route: " + code};
} }
} }
} else
}
std::cout << benchmark.name << std::endl;
std::cout << statistics << std::endl;
};
std::vector<Benchmark> benchmarks = {{"1000 matches, default radius"},
{"1000 matches, radius=10", 10},
{"1000 matches, radius=20", 20}};
for (const auto &benchmark : benchmarks)
{ {
run_benchmark(benchmark); statistics.push(TIMER_MSEC(match), iteration);
} }
});
} }
void runNearestBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations) void runNearestBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations)
@ -398,14 +429,20 @@ void runNearestBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int itera
std::optional<size_t> number_of_results = std::nullopt; std::optional<size_t> number_of_results = std::nullopt;
}; };
auto run_benchmark = [&](const Benchmark &benchmark) std::vector<Benchmark> benchmarks = {{"10000 nearest, number_of_results=1", 1},
{ {"10000 nearest, number_of_results=5", 5},
Statistics statistics{iterations}; {"10000 nearest, number_of_results=10", 10}};
auto NUM = 10000;
for (int iteration = 0; iteration < iterations; ++iteration) runBenchmarks(benchmarks,
{ iterations,
gpsTraces.resetSeed(); 10000,
for (int i = 0; i < NUM; ++i) osrm,
gpsTraces,
[](int iteration,
const Benchmark &benchmark,
const OSRM &osrm,
const GPSTraces &gpsTraces,
Statistics &statistics)
{ {
engine::api::ResultT result = json::Object(); engine::api::ResultT result = json::Object();
NearestParameters params; NearestParameters params;
@ -420,8 +457,6 @@ void runNearestBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int itera
const auto rc = osrm.Nearest(params, result); const auto rc = osrm.Nearest(params, result);
TIMER_STOP(nearest); TIMER_STOP(nearest);
statistics.push(TIMER_MSEC(nearest), iteration);
auto &json_result = std::get<json::Object>(result); auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok || if (rc != Status::Ok ||
json_result.values.find("waypoints") == json_result.values.end()) json_result.values.find("waypoints") == json_result.values.end())
@ -432,21 +467,11 @@ void runNearestBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int itera
throw std::runtime_error{"Couldn't find nearest point"}; throw std::runtime_error{"Couldn't find nearest point"};
} }
} }
} else
}
std::cout << benchmark.name << std::endl;
std::cout << statistics << std::endl;
};
std::vector<Benchmark> benchmarks = {{"10000 nearest, number_of_results=1", 1},
{"10000 nearest, number_of_results=5", 5},
{"10000 nearest, number_of_results=10", 10}};
for (const auto &benchmark : benchmarks)
{ {
run_benchmark(benchmark); statistics.push(TIMER_MSEC(nearest), iteration);
} }
});
} }
void runTripBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations) void runTripBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations)
@ -457,14 +482,21 @@ void runTripBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iteratio
size_t coordinates; size_t coordinates;
}; };
auto run_benchmark = [&](const Benchmark &benchmark) std::vector<Benchmark> benchmarks = {
{ {"250 trips, 3 coordinates", 3},
Statistics statistics{iterations}; {"250 trips, 5 coordinates", 5},
auto NUM = 250; };
for (int iteration = 0; iteration < iterations; ++iteration)
{ runBenchmarks(benchmarks,
gpsTraces.resetSeed(); iterations,
for (int i = 0; i < NUM; ++i) 250,
osrm,
gpsTraces,
[](int iteration,
const Benchmark &benchmark,
const OSRM &osrm,
const GPSTraces &gpsTraces,
Statistics &statistics)
{ {
engine::api::ResultT result = json::Object(); engine::api::ResultT result = json::Object();
TripParameters params; TripParameters params;
@ -479,8 +511,6 @@ void runTripBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iteratio
const auto rc = osrm.Trip(params, result); const auto rc = osrm.Trip(params, result);
TIMER_STOP(trip); TIMER_STOP(trip);
statistics.push(TIMER_MSEC(trip), iteration);
auto &json_result = std::get<json::Object>(result); auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok || if (rc != Status::Ok ||
json_result.values.find("trips") == json_result.values.end()) json_result.values.find("trips") == json_result.values.end())
@ -491,22 +521,11 @@ void runTripBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iteratio
throw std::runtime_error{"Couldn't find trip"}; throw std::runtime_error{"Couldn't find trip"};
} }
} }
} else
}
std::cout << benchmark.name << std::endl;
std::cout << statistics << std::endl;
};
std::vector<Benchmark> benchmarks = {
{"250 trips, 3 coordinates", 3},
{"250 trips, 5 coordinates", 5},
};
for (const auto &benchmark : benchmarks)
{ {
run_benchmark(benchmark); statistics.push(TIMER_MSEC(trip), iteration);
} }
});
} }
void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations) void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations)
{ {
@ -516,15 +535,20 @@ void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterati
size_t coordinates; size_t coordinates;
}; };
auto run_benchmark = [&](const Benchmark &benchmark) std::vector<Benchmark> benchmarks = {{"250 tables, 3 coordinates", 3},
{ {"250 tables, 25 coordinates", 25},
Statistics statistics{iterations}; {"250 tables, 50 coordinates", 50}};
auto NUM = 250;
for (int iteration = 0; iteration < iterations; ++iteration)
{
gpsTraces.resetSeed();
for (int i = 0; i < NUM; ++i) runBenchmarks(benchmarks,
iterations,
250,
osrm,
gpsTraces,
[](int iteration,
const Benchmark &benchmark,
const OSRM &osrm,
const GPSTraces &gpsTraces,
Statistics &statistics)
{ {
engine::api::ResultT result = json::Object(); engine::api::ResultT result = json::Object();
TableParameters params; TableParameters params;
@ -550,21 +574,7 @@ void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterati
throw std::runtime_error{"Couldn't compute table"}; throw std::runtime_error{"Couldn't compute table"};
} }
} }
} });
}
std::cout << benchmark.name << std::endl;
std::cout << statistics << std::endl;
};
std::vector<Benchmark> benchmarks = {{"250 tables, 3 coordinates", 3},
{"250 tables, 25 coordinates", 25},
{"250 tables, 50 coordinates", 50}};
for (const auto &benchmark : benchmarks)
{
run_benchmark(benchmark);
}
} }
} // namespace } // namespace