diff --git a/CMakeLists.txt b/CMakeLists.txt index 1876d6b85..bb9b4a3de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -357,7 +357,7 @@ if(ENABLE_CONAN) KEEP_RPATHS NO_OUTPUT_DIRS OPTIONS boost:filesystem_version=3 # https://stackoverflow.com/questions/73392648/error-with-boost-filesystem-version-in-cmake - onetbb:shared=${TBB_SHARED} + # onetbb:shared=${TBB_SHARED} boost:without_stacktrace=True # Apple Silicon cross-compilation fails without it BUILD missing ) diff --git a/src/benchmarks/route.cpp b/src/benchmarks/route.cpp index def90c175..275f2be71 100644 --- a/src/benchmarks/route.cpp +++ b/src/benchmarks/route.cpp @@ -1,3 +1,4 @@ +#include "engine/api/match_parameters.hpp" #include "engine/engine_config.hpp" #include "util/coordinate.hpp" #include "util/timing_util.hpp" @@ -20,33 +21,144 @@ #include #include #include +#include #include +#include +#include -int main(int argc, const char *argv[]) -try -{ - if (argc < 2) - { - std::cerr << "Usage: " << argv[0] << " data.osrm\n"; - return EXIT_FAILURE; +using namespace osrm; + +namespace { + + +class GPSTraces { +private: + std::set trackIDs; + std::unordered_map> traces; + std::vector coordinates; + mutable std::mt19937 gen; + +public: + GPSTraces(int seed) : gen(std::random_device{}()) { + gen.seed(seed); } - using namespace osrm; + bool readCSV(const std::string& filename) { + std::ifstream file(filename); + if (!file.is_open()) { + std::cerr << "Error opening file: " << filename << std::endl; + return false; + } - // Configure based on a .osrm base path, and no datasets in shared mem from osrm-datastore - EngineConfig config; - config.storage_config = {argv[1]}; - config.algorithm = (argc > 2 && std::string{argv[2]} == "mld") ? EngineConfig::Algorithm::MLD - : EngineConfig::Algorithm::CH; - config.use_shared_memory = false; + std::string line; + std::getline(file, line); - // Routing machine with several services (such as Route, Table, Nearest, Trip, Match) - OSRM osrm{config}; - struct Benchmark + while (std::getline(file, line)) { + std::istringstream ss(line); + std::string token; + + int trackID; + double latitude, longitude; + std::string time; + + std::getline(ss, token, ','); + trackID = std::stoi(token); + + std::getline(ss, token, ','); + latitude = std::stod(token); + + std::getline(ss, token, ','); + longitude = std::stod(token); + + // handle empty fields + if (std::getline(ss, token, ',')) { + time = token; + } + + trackIDs.insert(trackID); + traces[trackID].emplace_back(osrm::util::Coordinate{osrm::util::FloatLongitude{longitude}, osrm::util::FloatLatitude{latitude}}); + coordinates.emplace_back(osrm::util::Coordinate{osrm::util::FloatLongitude{longitude}, osrm::util::FloatLatitude{latitude}}); + } + + file.close(); + return true; + } + + const osrm::util::Coordinate& getRandomCoordinate() const { + std::uniform_int_distribution<> dis(0, coordinates.size() - 1); + return coordinates[dis(gen)]; + } + + std::vector getRandomTrace() const { + std::uniform_int_distribution<> dis(0, trackIDs.size() - 1); + auto it = trackIDs.begin(); + std::advance(it, dis(gen)); + auto trace = traces.at(*it); + + // slice trace + std::uniform_int_distribution<> dis2(0, trace.size() - 1); + auto start = dis2(gen); + auto end = dis2(gen); + if (start >= end) { + start = dis2(gen); + end = dis2(gen); + } + + return std::vector(trace.begin() + start, trace.begin() + end); + } +}; + +class Statistics { +public: + void push(double timeMs) { + times.push_back(timeMs); + sorted = false; + } + + double mean() { + return sum() / times.size(); + } + + double sum() { + double sum = 0; + for (auto time : times) { + sum += time; + } + return sum; + } + + double min() { + return *std::min_element(times.begin(), times.end()); + } + + double max() { + return *std::max_element(times.begin(), times.end()); + } + + double percentile(double p) { + const auto& times = getTimes(); + return times[static_cast(p * times.size())]; + } +private: + std::vector getTimes() { + if (!sorted) { + std::sort(times.begin(), times.end()); + sorted = true; + } + return times; + } + + std::vector times; + + bool sorted = false; +}; + +void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) { + struct Benchmark { std::string name; - std::vector coordinates; + size_t coordinates; RouteParameters::OverviewType overview; bool steps = false; std::optional alternatives = std::nullopt; @@ -58,7 +170,12 @@ try RouteParameters params; params.overview = benchmark.overview; params.steps = benchmark.steps; - params.coordinates = benchmark.coordinates; + + for (size_t i = 0; i < benchmark.coordinates; ++i) + { + params.coordinates.push_back(gpsTraces.getRandomCoordinate()); + } + if (benchmark.alternatives) { params.alternatives = *benchmark.alternatives; @@ -70,81 +187,80 @@ try params.coordinates.size(), boost::make_optional(*benchmark.radius)); } - TIMER_START(routes); - auto NUM = 1000; + Statistics statistics; + + auto NUM = 10000; for (int i = 0; i < NUM; ++i) { engine::api::ResultT result = json::Object(); + TIMER_START(routes); const auto rc = osrm.Route(params, result); + TIMER_STOP(routes); + + statistics.push(TIMER_MSEC(routes)); + auto &json_result = std::get(result); if (rc != Status::Ok || json_result.values.find("routes") == json_result.values.end()) { - throw std::runtime_error{"Couldn't route"}; + auto code = std::get(json_result.values["code"]).value; + if (code != "NoSegment") { + throw std::runtime_error{"Couldn't route"}; + } + } } - TIMER_STOP(routes); std::cout << benchmark.name << std::endl; - std::cout << TIMER_MSEC(routes) << "ms" << std::endl; - std::cout << TIMER_MSEC(routes) / NUM << "ms/req" << std::endl; + std::cout << "total: " << statistics.sum() << "ms" << std::endl; + std::cout << "avg: ~" << statistics.mean() << "ms/req" << std::endl; + std::cout << "p99: " << statistics.percentile(0.99) << "ms" << std::endl; + std::cout << "max: " << statistics.max() << "ms" << std::endl; }; std::vector benchmarks = { - {"1000 routes, 3 coordinates, no alternatives, overview=full, steps=true", - {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, - {FloatLongitude{7.421844922513342}, FloatLatitude{43.73690777888953}}, - {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, + {"10000 routes, 3 coordinates, no alternatives, overview=full, steps=true", + 3, RouteParameters::OverviewType::Full, true, std::nullopt}, - {"1000 routes, 2 coordinates, no alternatives, overview=full, steps=true", - {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, - {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, + {"10000 routes, 2 coordinates, no alternatives, overview=full, steps=true", + 2, RouteParameters::OverviewType::Full, true, std::nullopt}, - {"1000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true", - {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, - {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, + {"10000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true", + 2, RouteParameters::OverviewType::Full, true, 3}, - {"1000 routes, 3 coordinates, no alternatives, overview=false, steps=false", - {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, - {FloatLongitude{7.421844922513342}, FloatLatitude{43.73690777888953}}, - {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, + {"10000 routes, 3 coordinates, no alternatives, overview=false, steps=false", + 3, RouteParameters::OverviewType::False, false, std::nullopt}, - {"1000 routes, 2 coordinates, no alternatives, overview=false, steps=false", - {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, - {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, + {"10000 routes, 2 coordinates, no alternatives, overview=false, steps=false", + 2, RouteParameters::OverviewType::False, false, std::nullopt}, - {"1000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false", - {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, - {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, + {"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false", + 2, RouteParameters::OverviewType::False, false, 3}, - {"1000 routes, 3 coordinates, no alternatives, overview=false, steps=false, radius=750", - {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, - {FloatLongitude{7.421844922513342}, FloatLatitude{43.73690777888953}}, - {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, + {"10000 routes, 3 coordinates, no alternatives, overview=false, steps=false, radius=750", + 3, RouteParameters::OverviewType::False, false, std::nullopt, 750}, - {"1000 routes, 2 coordinates, no alternatives, overview=false, steps=false, radius=750", - {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, - {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, + {"10000 routes, 2 coordinates, no alternatives, overview=false, steps=false, radius=750", + 2, RouteParameters::OverviewType::False, false, std::nullopt, 750}, - {"1000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false, radius=750", - {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, - {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, + {"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false, radius=750", + 2, RouteParameters::OverviewType::False, false, 3, @@ -156,7 +272,107 @@ try { run_benchmark(benchmark); } +} +void runMatchBenchmark(const OSRM& osrm, const GPSTraces& gpsTraces) { + struct Benchmark { + std::string name; + std::optional radius = std::nullopt; + }; + + auto run_benchmark = [&](const Benchmark &benchmark) { + engine::api::MatchParameters params; + params.coordinates = gpsTraces.getRandomTrace(); + params.radiuses = {}; + if (benchmark.radius) + { + + std::cerr << "radiuses: " << params.coordinates.size() << std::endl; + for (size_t index = 0; index < params.coordinates.size(); ++index) + { + params.radiuses.emplace_back(*benchmark.radius); + } + } + + + Statistics statistics; + + auto NUM = 10000; + for (int i = 0; i < NUM; ++i) { + engine::api::ResultT result = json::Object(); + TIMER_START(match); + const auto rc = osrm.Match(params, result); + TIMER_STOP(match); + + statistics.push(TIMER_MSEC(match)); + + auto &json_result = std::get(result); + if (rc != Status::Ok || json_result.values.find("matchings") == json_result.values.end()) { + auto code = std::get(json_result.values["code"]).value; + if (code != "NoSegment") { + throw std::runtime_error{"Couldn't route"}; + } + } + } + + std::cout << benchmark.name << std::endl; + std::cout << "total: " << statistics.sum() << "ms" << std::endl; + std::cout << "avg: " << statistics.mean() << "ms/req" << std::endl; + std::cout << "p99: " << statistics.percentile(0.99) << "ms" << std::endl; + std::cout << "max: " << statistics.max() << "ms" << std::endl; + }; + + std::vector benchmarks = { + {"1000 matches, default radius"}, + {"1000 matches, radius=10"}, + // {"1000 matches, radius=20", 20} + }; + + for (const auto &benchmark : benchmarks) { + run_benchmark(benchmark); + } + +} + +// void runNearestBenchmark() { + +// } + +// void runTripBenchmark() { +// } + +// void runTableBenchmark() { +// } + +} // namespace + +int main(int argc, const char *argv[]) +try +{ + if (argc < 4) + { + std::cerr << "Usage: " << argv[0] << " data.osrm \n"; + return EXIT_FAILURE; + } + + + // Configure based on a .osrm base path, and no datasets in shared mem from osrm-datastore + EngineConfig config; + config.storage_config = {argv[1]}; + config.algorithm = std::string{argv[2]} == "mld" ? EngineConfig::Algorithm::MLD + : EngineConfig::Algorithm::CH; + config.use_shared_memory = false; + + // Routing machine with several services (such as Route, Table, Nearest, Trip, Match) + OSRM osrm{config}; + + GPSTraces gpsTraces{42}; + gpsTraces.readCSV(argv[3]); + + //runRouteBenchmark(osrm, gpsTraces); + (void)runRouteBenchmark; + + runMatchBenchmark(osrm, gpsTraces); return EXIT_SUCCESS; } catch (const std::exception &e)