Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

wip

Browse Source
This commit is contained in:
Siarhei Fedartsou 2024-06-16 21:42:29 +02:00
parent c6193a9d79
commit 930329b7aa
3 changed files with 77 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/ci/e2e_benchmark.py
View File

@ -130,11 +130,8 @@ def main():
assert all_times.shape == (args.iterations, all_times.shape[1])
print('Shape: ', all_times.shape)
total_time, total_ci, total_best = calculate_confidence_interval(np.sum(all_times, axis=1))
print('Ops: ', float(all_times.shape[1]) / np.sum(all_times * 1000, axis=1))
ops_per_sec, ops_per_sec_ci, ops_per_sec_best = calculate_confidence_interval(float(all_times.shape[1]) / np.sum(all_times / 1000, axis=1))
min_time, min_ci, _ = calculate_confidence_interval(np.min(all_times, axis=1))
mean_time, mean_ci, _ = calculate_confidence_interval(np.mean(all_times, axis=1))
@ -143,8 +140,8 @@ def main():
perc_99_time, perc_99_ci, _ = calculate_confidence_interval(np.percentile(all_times, 99, axis=1))
max_time, max_ci, _ = calculate_confidence_interval(np.max(all_times, axis=1))
print(f'Total: {total_time:.2f}ms ± {total_ci:.2f}ms. Best: {total_best:.2f}ms')
print(f'Ops: {ops_per_sec:.2f} ± {ops_per_sec_ci:.2f} ops/s. Best: {ops_per_sec_best:.2f} ops/s')
print(f'Total: {total_time:.2f}ms ± {total_ci:.2f}ms. Best: {total_best:.2f}ms')
print(f"Min time: {min_time:.2f}ms ± {min_ci:.2f}ms")
print(f"Mean time: {mean_time:.2f}ms ± {mean_ci:.2f}ms")
print(f"Median time: {median_time:.2f}ms ± {median_ci:.2f}ms")

44
scripts/ci/run_benchmarks.sh
View File

@ -53,22 +53,22 @@ function run_benchmarks_for_folder {
mkdir -p $RESULTS_FOLDER
BENCHMARKS_FOLDER="$BINARIES_FOLDER/src/benchmarks"
# echo "Running match-bench MLD"
# $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
# echo "Running match-bench CH"
# $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
# echo "Running route-bench MLD"
# $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
# echo "Running route-bench CH"
# $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
# echo "Running alias"
# $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
# echo "Running json-render-bench"
# $BENCHMARKS_FOLDER/json-render-bench "$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
# echo "Running packedvector-bench"
# $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
# echo "Running rtree-bench"
# $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
echo "Running match-bench MLD"
$BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
echo "Running match-bench CH"
$BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
echo "Running route-bench MLD"
$BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
echo "Running route-bench CH"
$BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
echo "Running alias"
$BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
echo "Running json-render-bench"
$BENCHMARKS_FOLDER/json-render-bench "$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
echo "Running packedvector-bench"
$BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
echo "Running rtree-bench"
$BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
cp -rf $OSM_PBF $FOLDER/data.osm.pbf
@ -81,12 +81,12 @@ function run_benchmarks_for_folder {
# echo "Running osrm-contract"
measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench"
# for BENCH in nearest table trip route match; do
# echo "Running random $BENCH MLD"
# $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" mld $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_mld.bench" || true
# echo "Running random $BENCH CH"
# $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" ch $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_ch.bench" || true
# done
for BENCH in nearest table trip route match; do
echo "Running random $BENCH MLD"
$BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" mld $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_mld.bench" 5 || true
echo "Running random $BENCH CH"
$BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" ch $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_ch.bench" 5 || true
done
for ALGORITHM in ch mld; do

69
src/benchmarks/bench.cpp
View File

@ -45,8 +45,12 @@ class GPSTraces
std::vector<osrm::util::Coordinate> coordinates;
mutable std::mt19937 gen;
int seed;
public:
GPSTraces(int seed) : gen(std::random_device{}()) { gen.seed(seed); }
GPSTraces(int seed) : gen(std::random_device{}()), seed(seed) { gen.seed(seed); }
void resetSeed() const { gen.seed(seed); }
bool readCSV(const std::string &filename)
{
@ -113,7 +117,7 @@ class GPSTraces
class Statistics
{
public:
void push(double timeMs)
void push(double timeMs, int /*iteration*/ = 0)
{
times.push_back(timeMs);
sorted = false;
@ -168,7 +172,7 @@ std::ostream &operator<<(std::ostream &os, Statistics &statistics)
return os;
}
void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations)
{
struct Benchmark
{
@ -184,7 +188,11 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
{
Statistics statistics;
auto NUM = 10000;
for (int iteration = 0; iteration < iterations; ++iteration)
{
gpsTraces.resetSeed();
auto NUM = 1000;
for (int i = 0; i < NUM; ++i)
{
RouteParameters params;
@ -212,10 +220,11 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
const auto rc = osrm.Route(params, result);
TIMER_STOP(routes);
statistics.push(TIMER_MSEC(routes));
statistics.push(TIMER_MSEC(routes), iteration);
auto &json_result = std::get<json::Object>(result);
if (rc != Status::Ok || json_result.values.find("routes") == json_result.values.end())
if (rc != Status::Ok ||
json_result.values.find("routes") == json_result.values.end())
{
auto code = std::get<json::String>(json_result.values["code"]).value;
if (code != "NoSegment" && code != "NoRoute")
@ -224,6 +233,7 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
}
}
}
}
std::cout << benchmark.name << std::endl;
std::cout << statistics << std::endl;
};
@ -234,11 +244,6 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
RouteParameters::OverviewType::Full,
true,
std::nullopt},
{"10000 routes, 2 coordinates, no alternatives, overview=full, steps=true",
2,
RouteParameters::OverviewType::Full,
true,
std::nullopt},
{"10000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true",
2,
RouteParameters::OverviewType::Full,
@ -249,36 +254,11 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
RouteParameters::OverviewType::False,
false,
std::nullopt},
{"10000 routes, 2 coordinates, no alternatives, overview=false, steps=false",
2,
RouteParameters::OverviewType::False,
false,
std::nullopt},
{"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false",
2,
RouteParameters::OverviewType::False,
false,
3},
{"10000 routes, 3 coordinates, no alternatives, overview=false, steps=false, radius=750",
3,
RouteParameters::OverviewType::False,
false,
std::nullopt,
750},
{"10000 routes, 2 coordinates, no alternatives, overview=false, steps=false, radius=750",
2,
RouteParameters::OverviewType::False,
false,
std::nullopt,
750},
{"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false, radius=750",
2,
RouteParameters::OverviewType::False,
false,
3,
750}
};
3}};
for (const auto &benchmark : benchmarks)
{
@ -502,8 +482,7 @@ void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
std::vector<Benchmark> benchmarks = {{"250 tables, 3 coordinates", 3},
{"250 tables, 25 coordinates", 25},
{"250 tables, 50 coordinates", 50},
{"250 tables, 100 coordinates", 100}};
{"250 tables, 50 coordinates", 50}};
for (const auto &benchmark : benchmarks)
{
@ -516,11 +495,11 @@ void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
int main(int argc, const char *argv[])
try
{
if (argc < 5)
if (argc < 6)
{
std::cerr
<< "Usage: " << argv[0]
<< " data.osrm <mld|ch> <path to GPS traces.csv> <route|match|trip|table|nearest>\n";
std::cerr << "Usage: " << argv[0]
<< " data.osrm <mld|ch> <path to GPS traces.csv> "
"<route|match|trip|table|nearest> <number_of_iterations>\n";
return EXIT_FAILURE;
}
@ -537,11 +516,13 @@ try
GPSTraces gpsTraces{42};
gpsTraces.readCSV(argv[3]);
int iterations = std::stoi(argv[5]);
const auto benchmarkToRun = std::string{argv[4]};
if (benchmarkToRun == "route")
{
runRouteBenchmark(osrm, gpsTraces);
runRouteBenchmark(osrm, gpsTraces, iterations);
}
else if (benchmarkToRun == "match")
{