wip
This commit is contained in:
Siarhei Fedartsou
parent
c6193a9d79
commit
930329b7aa
@ -130,11 +130,8 @@ def main():
|
||||
|
||||
assert all_times.shape == (args.iterations, all_times.shape[1])
|
||||
|
||||
print('Shape: ', all_times.shape)
|
||||
|
||||
total_time, total_ci, total_best = calculate_confidence_interval(np.sum(all_times, axis=1))
|
||||
print('Ops: ', float(all_times.shape[1]) / np.sum(all_times * 1000, axis=1))
|
||||
|
||||
ops_per_sec, ops_per_sec_ci, ops_per_sec_best = calculate_confidence_interval(float(all_times.shape[1]) / np.sum(all_times / 1000, axis=1))
|
||||
min_time, min_ci, _ = calculate_confidence_interval(np.min(all_times, axis=1))
|
||||
mean_time, mean_ci, _ = calculate_confidence_interval(np.mean(all_times, axis=1))
|
||||
@ -143,8 +140,8 @@ def main():
|
||||
perc_99_time, perc_99_ci, _ = calculate_confidence_interval(np.percentile(all_times, 99, axis=1))
|
||||
max_time, max_ci, _ = calculate_confidence_interval(np.max(all_times, axis=1))
|
||||
|
||||
print(f'Total: {total_time:.2f}ms ± {total_ci:.2f}ms. Best: {total_best:.2f}ms')
|
||||
print(f'Ops: {ops_per_sec:.2f} ± {ops_per_sec_ci:.2f} ops/s. Best: {ops_per_sec_best:.2f} ops/s')
|
||||
print(f'Total: {total_time:.2f}ms ± {total_ci:.2f}ms. Best: {total_best:.2f}ms')
|
||||
print(f"Min time: {min_time:.2f}ms ± {min_ci:.2f}ms")
|
||||
print(f"Mean time: {mean_time:.2f}ms ± {mean_ci:.2f}ms")
|
||||
print(f"Median time: {median_time:.2f}ms ± {median_ci:.2f}ms")
|
||||
|
||||
@ -53,22 +53,22 @@ function run_benchmarks_for_folder {
|
||||
mkdir -p $RESULTS_FOLDER
|
||||
|
||||
BENCHMARKS_FOLDER="$BINARIES_FOLDER/src/benchmarks"
|
||||
# echo "Running match-bench MLD"
|
||||
# $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
|
||||
# echo "Running match-bench CH"
|
||||
# $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
|
||||
# echo "Running route-bench MLD"
|
||||
# $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
|
||||
# echo "Running route-bench CH"
|
||||
# $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
|
||||
# echo "Running alias"
|
||||
# $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
|
||||
# echo "Running json-render-bench"
|
||||
# $BENCHMARKS_FOLDER/json-render-bench "$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
|
||||
# echo "Running packedvector-bench"
|
||||
# $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
|
||||
# echo "Running rtree-bench"
|
||||
# $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
|
||||
echo "Running match-bench MLD"
|
||||
$BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
|
||||
echo "Running match-bench CH"
|
||||
$BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench"
|
||||
echo "Running route-bench MLD"
|
||||
$BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench"
|
||||
echo "Running route-bench CH"
|
||||
$BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench"
|
||||
echo "Running alias"
|
||||
$BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench"
|
||||
echo "Running json-render-bench"
|
||||
$BENCHMARKS_FOLDER/json-render-bench "$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench"
|
||||
echo "Running packedvector-bench"
|
||||
$BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench"
|
||||
echo "Running rtree-bench"
|
||||
$BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench"
|
||||
|
||||
cp -rf $OSM_PBF $FOLDER/data.osm.pbf
|
||||
|
||||
@ -81,12 +81,12 @@ function run_benchmarks_for_folder {
|
||||
# echo "Running osrm-contract"
|
||||
measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench"
|
||||
|
||||
# for BENCH in nearest table trip route match; do
|
||||
# echo "Running random $BENCH MLD"
|
||||
# $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" mld $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_mld.bench" || true
|
||||
# echo "Running random $BENCH CH"
|
||||
# $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" ch $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_ch.bench" || true
|
||||
# done
|
||||
for BENCH in nearest table trip route match; do
|
||||
echo "Running random $BENCH MLD"
|
||||
$BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" mld $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_mld.bench" 5 || true
|
||||
echo "Running random $BENCH CH"
|
||||
$BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" ch $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_ch.bench" 5 || true
|
||||
done
|
||||
|
||||
|
||||
for ALGORITHM in ch mld; do
|
||||
|
||||
@ -45,8 +45,12 @@ class GPSTraces
|
||||
std::vector<osrm::util::Coordinate> coordinates;
|
||||
mutable std::mt19937 gen;
|
||||
|
||||
int seed;
|
||||
|
||||
public:
|
||||
GPSTraces(int seed) : gen(std::random_device{}()) { gen.seed(seed); }
|
||||
GPSTraces(int seed) : gen(std::random_device{}()), seed(seed) { gen.seed(seed); }
|
||||
|
||||
void resetSeed() const { gen.seed(seed); }
|
||||
|
||||
bool readCSV(const std::string &filename)
|
||||
{
|
||||
@ -113,7 +117,7 @@ class GPSTraces
|
||||
class Statistics
|
||||
{
|
||||
public:
|
||||
void push(double timeMs)
|
||||
void push(double timeMs, int /*iteration*/ = 0)
|
||||
{
|
||||
times.push_back(timeMs);
|
||||
sorted = false;
|
||||
@ -168,7 +172,7 @@ std::ostream &operator<<(std::ostream &os, Statistics &statistics)
|
||||
return os;
|
||||
}
|
||||
|
||||
void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
|
||||
void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations)
|
||||
{
|
||||
struct Benchmark
|
||||
{
|
||||
@ -184,43 +188,49 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
|
||||
{
|
||||
Statistics statistics;
|
||||
|
||||
auto NUM = 10000;
|
||||
for (int i = 0; i < NUM; ++i)
|
||||
for (int iteration = 0; iteration < iterations; ++iteration)
|
||||
{
|
||||
RouteParameters params;
|
||||
params.overview = benchmark.overview;
|
||||
params.steps = benchmark.steps;
|
||||
gpsTraces.resetSeed();
|
||||
|
||||
for (size_t i = 0; i < benchmark.coordinates; ++i)
|
||||
auto NUM = 1000;
|
||||
for (int i = 0; i < NUM; ++i)
|
||||
{
|
||||
params.coordinates.push_back(gpsTraces.getRandomCoordinate());
|
||||
}
|
||||
RouteParameters params;
|
||||
params.overview = benchmark.overview;
|
||||
params.steps = benchmark.steps;
|
||||
|
||||
if (benchmark.alternatives)
|
||||
{
|
||||
params.alternatives = *benchmark.alternatives;
|
||||
}
|
||||
|
||||
if (benchmark.radius)
|
||||
{
|
||||
params.radiuses = std::vector<boost::optional<double>>(
|
||||
params.coordinates.size(), boost::make_optional(*benchmark.radius));
|
||||
}
|
||||
|
||||
engine::api::ResultT result = json::Object();
|
||||
TIMER_START(routes);
|
||||
const auto rc = osrm.Route(params, result);
|
||||
TIMER_STOP(routes);
|
||||
|
||||
statistics.push(TIMER_MSEC(routes));
|
||||
|
||||
auto &json_result = std::get<json::Object>(result);
|
||||
if (rc != Status::Ok || json_result.values.find("routes") == json_result.values.end())
|
||||
{
|
||||
auto code = std::get<json::String>(json_result.values["code"]).value;
|
||||
if (code != "NoSegment" && code != "NoRoute")
|
||||
for (size_t i = 0; i < benchmark.coordinates; ++i)
|
||||
{
|
||||
throw std::runtime_error{"Couldn't route: " + code};
|
||||
params.coordinates.push_back(gpsTraces.getRandomCoordinate());
|
||||
}
|
||||
|
||||
if (benchmark.alternatives)
|
||||
{
|
||||
params.alternatives = *benchmark.alternatives;
|
||||
}
|
||||
|
||||
if (benchmark.radius)
|
||||
{
|
||||
params.radiuses = std::vector<boost::optional<double>>(
|
||||
params.coordinates.size(), boost::make_optional(*benchmark.radius));
|
||||
}
|
||||
|
||||
engine::api::ResultT result = json::Object();
|
||||
TIMER_START(routes);
|
||||
const auto rc = osrm.Route(params, result);
|
||||
TIMER_STOP(routes);
|
||||
|
||||
statistics.push(TIMER_MSEC(routes), iteration);
|
||||
|
||||
auto &json_result = std::get<json::Object>(result);
|
||||
if (rc != Status::Ok ||
|
||||
json_result.values.find("routes") == json_result.values.end())
|
||||
{
|
||||
auto code = std::get<json::String>(json_result.values["code"]).value;
|
||||
if (code != "NoSegment" && code != "NoRoute")
|
||||
{
|
||||
throw std::runtime_error{"Couldn't route: " + code};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -234,11 +244,6 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
|
||||
RouteParameters::OverviewType::Full,
|
||||
true,
|
||||
std::nullopt},
|
||||
{"10000 routes, 2 coordinates, no alternatives, overview=full, steps=true",
|
||||
2,
|
||||
RouteParameters::OverviewType::Full,
|
||||
true,
|
||||
std::nullopt},
|
||||
{"10000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true",
|
||||
2,
|
||||
RouteParameters::OverviewType::Full,
|
||||
@ -249,36 +254,11 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
|
||||
RouteParameters::OverviewType::False,
|
||||
false,
|
||||
std::nullopt},
|
||||
{"10000 routes, 2 coordinates, no alternatives, overview=false, steps=false",
|
||||
2,
|
||||
RouteParameters::OverviewType::False,
|
||||
false,
|
||||
std::nullopt},
|
||||
{"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false",
|
||||
2,
|
||||
RouteParameters::OverviewType::False,
|
||||
false,
|
||||
3},
|
||||
{"10000 routes, 3 coordinates, no alternatives, overview=false, steps=false, radius=750",
|
||||
3,
|
||||
RouteParameters::OverviewType::False,
|
||||
false,
|
||||
std::nullopt,
|
||||
750},
|
||||
{"10000 routes, 2 coordinates, no alternatives, overview=false, steps=false, radius=750",
|
||||
2,
|
||||
RouteParameters::OverviewType::False,
|
||||
false,
|
||||
std::nullopt,
|
||||
750},
|
||||
{"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false, radius=750",
|
||||
2,
|
||||
RouteParameters::OverviewType::False,
|
||||
false,
|
||||
3,
|
||||
750}
|
||||
|
||||
};
|
||||
3}};
|
||||
|
||||
for (const auto &benchmark : benchmarks)
|
||||
{
|
||||
@ -502,8 +482,7 @@ void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
|
||||
|
||||
std::vector<Benchmark> benchmarks = {{"250 tables, 3 coordinates", 3},
|
||||
{"250 tables, 25 coordinates", 25},
|
||||
{"250 tables, 50 coordinates", 50},
|
||||
{"250 tables, 100 coordinates", 100}};
|
||||
{"250 tables, 50 coordinates", 50}};
|
||||
|
||||
for (const auto &benchmark : benchmarks)
|
||||
{
|
||||
@ -516,11 +495,11 @@ void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces)
|
||||
int main(int argc, const char *argv[])
|
||||
try
|
||||
{
|
||||
if (argc < 5)
|
||||
if (argc < 6)
|
||||
{
|
||||
std::cerr
|
||||
<< "Usage: " << argv[0]
|
||||
<< " data.osrm <mld|ch> <path to GPS traces.csv> <route|match|trip|table|nearest>\n";
|
||||
std::cerr << "Usage: " << argv[0]
|
||||
<< " data.osrm <mld|ch> <path to GPS traces.csv> "
|
||||
"<route|match|trip|table|nearest> <number_of_iterations>\n";
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
@ -537,11 +516,13 @@ try
|
||||
GPSTraces gpsTraces{42};
|
||||
gpsTraces.readCSV(argv[3]);
|
||||
|
||||
int iterations = std::stoi(argv[5]);
|
||||
|
||||
const auto benchmarkToRun = std::string{argv[4]};
|
||||
|
||||
if (benchmarkToRun == "route")
|
||||
{
|
||||
runRouteBenchmark(osrm, gpsTraces);
|
||||
runRouteBenchmark(osrm, gpsTraces, iterations);
|
||||
}
|
||||
else if (benchmarkToRun == "match")
|
||||
{
|
||||
|
||||
Loading…
Reference in New Issue
Block a user