From 054161fc7e9c392c9d0788529e431d9267938681 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Mon, 10 Jun 2024 20:06:43 +0200 Subject: [PATCH 01/17] Avoid copy of vectors in MakeRoute function (#6939) --- CHANGELOG.md | 1 + include/engine/api/route_api.hpp | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad9e06bf9..d59d4c6e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - NodeJS: - CHANGED: Use node-api instead of NAN. [#6452](https://github.com/Project-OSRM/osrm-backend/pull/6452) - Misc: + - CHANGED: Avoid copy of vectors in MakeRoute function. [#6939](https://github.com/Project-OSRM/osrm-backend/pull/6939) - FIXED: Fix bugprone-unused-return-value clang-tidy warning. [#6934](https://github.com/Project-OSRM/osrm-backend/pull/6934) - FIXED: Fix performance-noexcept-move-constructor clang-tidy warning. [#6931](https://github.com/Project-OSRM/osrm-backend/pull/6933) - FIXED: Fix performance-noexcept-swap clang-tidy warning. [#6931](https://github.com/Project-OSRM/osrm-backend/pull/6931) diff --git a/include/engine/api/route_api.hpp b/include/engine/api/route_api.hpp index f833d603c..b6907ddca 100644 --- a/include/engine/api/route_api.hpp +++ b/include/engine/api/route_api.hpp @@ -340,8 +340,8 @@ class RouteAPI : public BaseAPI unpacked_path_segments, source_traversed_in_reverse, target_traversed_in_reverse); - std::vector legs = legs_info.first; - std::vector leg_geometries = legs_info.second; + std::vector &legs = legs_info.first; + std::vector &leg_geometries = legs_info.second; auto route = guidance::assembleRoute(legs); // Fill legs @@ -716,8 +716,8 @@ class RouteAPI : public BaseAPI unpacked_path_segments, source_traversed_in_reverse, target_traversed_in_reverse); - std::vector legs = legs_info.first; - std::vector leg_geometries = legs_info.second; + std::vector &legs = legs_info.first; + std::vector &leg_geometries = legs_info.second; auto route = guidance::assembleRoute(legs); boost::optional json_overview = From 0188d2bccd45e724082559155d7637389ccb8ce8 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Mon, 10 Jun 2024 21:29:31 +0200 Subject: [PATCH 02/17] Use std::unordered_map::emplace instead of operator[] when producing JSONs (#6936) --- CHANGELOG.md | 1 + src/engine/api/json_factory.cpp | 117 ++++++++++++++++++-------------- 2 files changed, 68 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d59d4c6e8..52733b5b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - NodeJS: - CHANGED: Use node-api instead of NAN. [#6452](https://github.com/Project-OSRM/osrm-backend/pull/6452) - Misc: + - CHANGED: Use std::unordered_map::emplace instead of operator[] when producing JSONs. [#6936](https://github.com/Project-OSRM/osrm-backend/pull/6936) - CHANGED: Avoid copy of vectors in MakeRoute function. [#6939](https://github.com/Project-OSRM/osrm-backend/pull/6939) - FIXED: Fix bugprone-unused-return-value clang-tidy warning. [#6934](https://github.com/Project-OSRM/osrm-backend/pull/6934) - FIXED: Fix performance-noexcept-move-constructor clang-tidy warning. [#6931](https://github.com/Project-OSRM/osrm-backend/pull/6933) diff --git a/src/engine/api/json_factory.cpp b/src/engine/api/json_factory.cpp index 673de0496..6765c9351 100644 --- a/src/engine/api/json_factory.cpp +++ b/src/engine/api/json_factory.cpp @@ -44,21 +44,22 @@ util::json::Array lanesFromIntersection(const guidance::IntermediateIntersection { BOOST_ASSERT(intersection.lanes.lanes_in_turn >= 1); util::json::Array result; + result.values.reserve(intersection.lane_description.size()); LaneID lane_id = intersection.lane_description.size(); for (const auto &lane_desc : intersection.lane_description) { --lane_id; util::json::Object lane; - lane.values["indications"] = toJSON(lane_desc); + lane.values.emplace("indications", toJSON(lane_desc)); if (lane_id >= intersection.lanes.first_lane_from_the_right && lane_id < intersection.lanes.first_lane_from_the_right + intersection.lanes.lanes_in_turn) - lane.values["valid"] = util::json::True(); + lane.values.emplace("valid", util::json::True()); else - lane.values["valid"] = util::json::False(); + lane.values.emplace("valid", util::json::False()); - result.values.push_back(lane); + result.values.emplace_back(std::move(lane)); } return result; @@ -77,6 +78,7 @@ std::string waypointTypeToString(const guidance::WaypointType waypoint_type) util::json::Value coordinateToLonLat(const util::Coordinate &coordinate) { util::json::Array array; + array.values.reserve(2); array.values.push_back(static_cast(util::toFloating(coordinate.lon))); array.values.push_back(static_cast(util::toFloating(coordinate.lat))); return util::json::Value{std::move(array)}; @@ -98,17 +100,20 @@ util::json::Object makeStepManeuver(const guidance::StepManeuver &maneuver) // These invalid responses should never happen: log if they do happen BOOST_ASSERT_MSG(maneuver_type != "invalid", "unexpected invalid maneuver type"); - step_maneuver.values["type"] = std::move(maneuver_type); + step_maneuver.values.emplace("type", std::move(maneuver_type)); if (detail::isValidModifier(maneuver)) - step_maneuver.values["modifier"] = - osrm::guidance::instructionModifierToString(maneuver.instruction.direction_modifier); + step_maneuver.values.emplace( + "modifier", + osrm::guidance::instructionModifierToString(maneuver.instruction.direction_modifier)); - step_maneuver.values["location"] = detail::coordinateToLonLat(maneuver.location); - step_maneuver.values["bearing_before"] = detail::roundAndClampBearing(maneuver.bearing_before); - step_maneuver.values["bearing_after"] = detail::roundAndClampBearing(maneuver.bearing_after); + step_maneuver.values.emplace("location", detail::coordinateToLonLat(maneuver.location)); + step_maneuver.values.emplace("bearing_before", + detail::roundAndClampBearing(maneuver.bearing_before)); + step_maneuver.values.emplace("bearing_after", + detail::roundAndClampBearing(maneuver.bearing_after)); if (maneuver.exit != 0) - step_maneuver.values["exit"] = maneuver.exit; + step_maneuver.values.emplace("exit", maneuver.exit); return step_maneuver; } @@ -137,16 +142,16 @@ util::json::Object makeIntersection(const guidance::IntermediateIntersection &in return util::json::False(); }); - result.values["location"] = detail::coordinateToLonLat(intersection.location); - result.values["bearings"] = bearings; - result.values["entry"] = entry; + result.values.emplace("location", detail::coordinateToLonLat(intersection.location)); + result.values.emplace("bearings", bearings); + result.values.emplace("entry", entry); if (intersection.in != guidance::IntermediateIntersection::NO_INDEX) - result.values["in"] = intersection.in; + result.values.emplace("in", intersection.in); if (intersection.out != guidance::IntermediateIntersection::NO_INDEX) - result.values["out"] = intersection.out; + result.values.emplace("out", intersection.out); if (detail::hasValidLanes(intersection)) - result.values["lanes"] = detail::lanesFromIntersection(intersection); + result.values.emplace("lanes", detail::lanesFromIntersection(intersection)); if (!intersection.classes.empty()) { @@ -157,7 +162,7 @@ util::json::Object makeIntersection(const guidance::IntermediateIntersection &in std::back_inserter(classes.values), [](const std::string &class_name) { return util::json::String{class_name}; }); - result.values["classes"] = std::move(classes); + result.values.emplace("classes", std::move(classes)); } return result; @@ -166,39 +171,44 @@ util::json::Object makeIntersection(const guidance::IntermediateIntersection &in util::json::Object makeRouteStep(guidance::RouteStep step, util::json::Value geometry) { util::json::Object route_step; - route_step.values["distance"] = std::round(step.distance * 10) / 10.; - route_step.values["duration"] = step.duration; - route_step.values["weight"] = step.weight; - route_step.values["name"] = std::move(step.name); + route_step.values.reserve(15); + + route_step.values.emplace("distance", std::round(step.distance * 10) / 10.); + route_step.values.emplace("duration", step.duration); + route_step.values.emplace("weight", step.weight); + route_step.values.emplace("name", step.name); + if (!step.ref.empty()) - route_step.values["ref"] = std::move(step.ref); + route_step.values.emplace("ref", step.ref); if (!step.pronunciation.empty()) - route_step.values["pronunciation"] = std::move(step.pronunciation); + route_step.values.emplace("pronunciation", step.pronunciation); if (!step.destinations.empty()) - route_step.values["destinations"] = std::move(step.destinations); + route_step.values.emplace("destinations", step.destinations); if (!step.exits.empty()) - route_step.values["exits"] = std::move(step.exits); + route_step.values.emplace("exits", step.exits); if (!step.rotary_name.empty()) { - route_step.values["rotary_name"] = std::move(step.rotary_name); + route_step.values.emplace("rotary_name", step.rotary_name); if (!step.rotary_pronunciation.empty()) { - route_step.values["rotary_pronunciation"] = std::move(step.rotary_pronunciation); + route_step.values.emplace("rotary_pronunciation", step.rotary_pronunciation); } } - route_step.values["mode"] = extractor::travelModeToString(step.mode); - route_step.values["maneuver"] = makeStepManeuver(step.maneuver); - route_step.values["geometry"] = std::move(geometry); - route_step.values["driving_side"] = step.is_left_hand_driving ? "left" : "right"; + route_step.values.emplace("mode", extractor::travelModeToString(step.mode)); + route_step.values.emplace("maneuver", makeStepManeuver(step.maneuver)); + route_step.values.emplace("geometry", std::move(geometry)); + route_step.values.emplace("driving_side", step.is_left_hand_driving ? "left" : "right"); util::json::Array intersections; intersections.values.reserve(step.intersections.size()); + std::transform(step.intersections.begin(), step.intersections.end(), std::back_inserter(intersections.values), makeIntersection); - route_step.values["intersections"] = std::move(intersections); + + route_step.values.emplace("intersections", std::move(intersections)); return route_step; } @@ -209,14 +219,16 @@ util::json::Object makeRoute(const guidance::Route &route, const char *weight_name) { util::json::Object json_route; - json_route.values["distance"] = route.distance; - json_route.values["duration"] = route.duration; - json_route.values["weight"] = route.weight; - json_route.values["weight_name"] = weight_name; - json_route.values["legs"] = std::move(legs); + json_route.values.reserve(6); + + json_route.values.emplace("distance", route.distance); + json_route.values.emplace("duration", route.duration); + json_route.values.emplace("weight", route.weight); + json_route.values.emplace("weight_name", weight_name); + json_route.values.emplace("legs", std::move(legs)); if (geometry) { - json_route.values["geometry"] = *std::move(geometry); + json_route.values.emplace("geometry", *std::move(geometry)); } return json_route; } @@ -225,9 +237,11 @@ util::json::Object makeWaypoint(const util::Coordinate &location, const double &distance, std::string name) { util::json::Object waypoint; - waypoint.values["location"] = detail::coordinateToLonLat(location); - waypoint.values["name"] = std::move(name); - waypoint.values["distance"] = distance; + waypoint.values.reserve(3); + + waypoint.values.emplace("location", detail::coordinateToLonLat(location)); + waypoint.values.emplace("name", std::move(name)); + waypoint.values.emplace("distance", distance); return waypoint; } @@ -237,26 +251,29 @@ util::json::Object makeWaypoint(const util::Coordinate &location, const Hint &location_hints) { auto waypoint = makeWaypoint(location, distance, std::move(name)); - waypoint.values["hint"] = location_hints.ToBase64(); + waypoint.values.reserve(1); + waypoint.values.emplace("hint", location_hints.ToBase64()); return waypoint; } util::json::Object makeRouteLeg(guidance::RouteLeg leg, util::json::Array steps) { util::json::Object route_leg; - route_leg.values["distance"] = leg.distance; - route_leg.values["duration"] = leg.duration; - route_leg.values["weight"] = leg.weight; - route_leg.values["summary"] = std::move(leg.summary); - route_leg.values["steps"] = std::move(steps); + route_leg.values.reserve(5); + + route_leg.values.emplace("distance", leg.distance); + route_leg.values.emplace("duration", leg.duration); + route_leg.values.emplace("weight", leg.weight); + route_leg.values.emplace("summary", std::move(leg.summary)); + route_leg.values.emplace("steps", std::move(steps)); return route_leg; } - util::json::Object makeRouteLeg(guidance::RouteLeg leg, util::json::Array steps, util::json::Object annotation) { util::json::Object route_leg = makeRouteLeg(std::move(leg), std::move(steps)); - route_leg.values["annotation"] = std::move(annotation); + route_leg.values.reserve(1); + route_leg.values.emplace("annotation", std::move(annotation)); return route_leg; } From de2f39296053412ec2f5668f2e7362f3cfbc4da8 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Mon, 10 Jun 2024 22:40:37 +0200 Subject: [PATCH 03/17] Avoid copy of intersection in totalTurnAngle (#6938) --- CHANGELOG.md | 1 + include/engine/guidance/collapsing_utility.hpp | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52733b5b9..cc75331bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - NodeJS: - CHANGED: Use node-api instead of NAN. [#6452](https://github.com/Project-OSRM/osrm-backend/pull/6452) - Misc: + - CHANGED: Avoid copy of intersection in totalTurnAngle. [#6938](https://github.com/Project-OSRM/osrm-backend/pull/6938) - CHANGED: Use std::unordered_map::emplace instead of operator[] when producing JSONs. [#6936](https://github.com/Project-OSRM/osrm-backend/pull/6936) - CHANGED: Avoid copy of vectors in MakeRoute function. [#6939](https://github.com/Project-OSRM/osrm-backend/pull/6939) - FIXED: Fix bugprone-unused-return-value clang-tidy warning. [#6934](https://github.com/Project-OSRM/osrm-backend/pull/6934) diff --git a/include/engine/guidance/collapsing_utility.hpp b/include/engine/guidance/collapsing_utility.hpp index 9177ef8d4..174dba019 100644 --- a/include/engine/guidance/collapsing_utility.hpp +++ b/include/engine/guidance/collapsing_utility.hpp @@ -202,8 +202,8 @@ inline double totalTurnAngle(const RouteStep &entry_step, const RouteStep &exit_ if (entry_step.geometry_begin > exit_step.geometry_begin) return totalTurnAngle(exit_step, entry_step); - const auto exit_intersection = exit_step.intersections.front(); - const auto entry_intersection = entry_step.intersections.front(); + const auto &exit_intersection = exit_step.intersections.front(); + const auto &entry_intersection = entry_step.intersections.front(); if ((exit_intersection.out >= exit_intersection.bearings.size()) || (entry_intersection.in >= entry_intersection.bearings.size())) return entry_intersection.bearings[entry_intersection.out]; From aa4e6b1cf332db07dbccf3eaaa6529b83842e81f Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Thu, 13 Jun 2024 18:18:36 +0200 Subject: [PATCH 04/17] Apply micro-optimisation for Nearest API (#6944) --- CHANGELOG.md | 1 + include/engine/api/nearest_api.hpp | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc75331bd..2edfeded4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - NodeJS: - CHANGED: Use node-api instead of NAN. [#6452](https://github.com/Project-OSRM/osrm-backend/pull/6452) - Misc: + - CHANGED: Apply micro-optimisation for Nearest API. [#6944](https://github.com/Project-OSRM/osrm-backend/pull/6944) - CHANGED: Avoid copy of intersection in totalTurnAngle. [#6938](https://github.com/Project-OSRM/osrm-backend/pull/6938) - CHANGED: Use std::unordered_map::emplace instead of operator[] when producing JSONs. [#6936](https://github.com/Project-OSRM/osrm-backend/pull/6936) - CHANGED: Avoid copy of vectors in MakeRoute function. [#6939](https://github.com/Project-OSRM/osrm-backend/pull/6939) diff --git a/include/engine/api/nearest_api.hpp b/include/engine/api/nearest_api.hpp index 32a5898d3..beae38cec 100644 --- a/include/engine/api/nearest_api.hpp +++ b/include/engine/api/nearest_api.hpp @@ -100,23 +100,23 @@ class NearestAPI final : public BaseAPI auto waypoint = MakeWaypoint({phantom_node}); util::json::Array nodes; + nodes.values.reserve(2); auto node_values = MakeNodes(phantom_node); - nodes.values.push_back(node_values.first); - nodes.values.push_back(node_values.second); - waypoint.values["nodes"] = std::move(nodes); - + nodes.values.emplace_back(node_values.first); + nodes.values.emplace_back(node_values.second); + waypoint.values.emplace("nodes", std::move(nodes)); return waypoint; }); - response.values["waypoints"] = std::move(waypoints); + response.values.emplace("waypoints", std::move(waypoints)); } - response.values["code"] = "Ok"; + response.values.emplace("code", "Ok"); auto data_timestamp = facade.GetTimestamp(); if (!data_timestamp.empty()) { - response.values["data_version"] = data_timestamp; + response.values.emplace("data_version", data_timestamp); } } From 7652f6ca6bcd8e21e9768a0064efc9debedb7889 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Fri, 14 Jun 2024 16:04:19 +0200 Subject: [PATCH 05/17] Apply micro-optimisation for Match API (#6945) --- CHANGELOG.md | 1 + include/engine/api/match_api.hpp | 34 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2edfeded4..2bd3782cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - NodeJS: - CHANGED: Use node-api instead of NAN. [#6452](https://github.com/Project-OSRM/osrm-backend/pull/6452) - Misc: + - CHANGED: Apply micro-optimisation for Match API. [#6945](https://github.com/Project-OSRM/osrm-backend/pull/6945) - CHANGED: Apply micro-optimisation for Nearest API. [#6944](https://github.com/Project-OSRM/osrm-backend/pull/6944) - CHANGED: Avoid copy of intersection in totalTurnAngle. [#6938](https://github.com/Project-OSRM/osrm-backend/pull/6938) - CHANGED: Use std::unordered_map::emplace instead of operator[] when producing JSONs. [#6936](https://github.com/Project-OSRM/osrm-backend/pull/6936) diff --git a/include/engine/api/match_api.hpp b/include/engine/api/match_api.hpp index 012c01176..d7956bba4 100644 --- a/include/engine/api/match_api.hpp +++ b/include/engine/api/match_api.hpp @@ -77,19 +77,19 @@ class MatchAPI final : public RouteAPI sub_routes[index].unpacked_path_segments, sub_routes[index].source_traversed_in_reverse, sub_routes[index].target_traversed_in_reverse); - route.values["confidence"] = sub_matchings[index].confidence; - routes.values.push_back(std::move(route)); + route.values.emplace("confidence", sub_matchings[index].confidence); + routes.values.emplace_back(std::move(route)); } if (!parameters.skip_waypoints) { - response.values["tracepoints"] = MakeTracepoints(sub_matchings); + response.values.emplace("tracepoints", MakeTracepoints(sub_matchings)); } - response.values["matchings"] = std::move(routes); - response.values["code"] = "Ok"; + response.values.emplace("matchings", std::move(routes)); + response.values.emplace("code", "Ok"); auto data_timestamp = facade.GetTimestamp(); if (!data_timestamp.empty()) { - response.values["data_version"] = data_timestamp; + response.values.emplace("data_version", data_timestamp); } } @@ -132,13 +132,13 @@ class MatchAPI final : public RouteAPI if (tidy_result.can_be_removed[trace_index]) { - waypoints.push_back(fbresult::WaypointBuilder(fb_result).Finish()); + waypoints.emplace_back(fbresult::WaypointBuilder(fb_result).Finish()); continue; } auto matching_index = trace_idx_to_matching_idx[trace_index]; if (matching_index.NotMatched()) { - waypoints.push_back(fbresult::WaypointBuilder(fb_result).Finish()); + waypoints.emplace_back(fbresult::WaypointBuilder(fb_result).Finish()); continue; } const auto &phantom = @@ -165,7 +165,7 @@ class MatchAPI final : public RouteAPI { waypoint->add_waypoint_index(matching_index.point_index); } - waypoints.push_back(waypoint->Finish()); + waypoints.emplace_back(waypoint->Finish()); } return fb_result.CreateVector(waypoints); @@ -186,23 +186,23 @@ class MatchAPI final : public RouteAPI { if (tidy_result.can_be_removed[trace_index]) { - waypoints.values.push_back(util::json::Null()); + waypoints.values.emplace_back(util::json::Null()); continue; } auto matching_index = trace_idx_to_matching_idx[trace_index]; if (matching_index.NotMatched()) { - waypoints.values.push_back(util::json::Null()); + waypoints.values.emplace_back(util::json::Null()); continue; } const auto &phantom = sub_matchings[matching_index.sub_matching_index].nodes[matching_index.point_index]; auto waypoint = BaseAPI::MakeWaypoint({phantom}); - waypoint.values["matchings_index"] = matching_index.sub_matching_index; - waypoint.values["waypoint_index"] = matching_index.point_index; - waypoint.values["alternatives_count"] = - sub_matchings[matching_index.sub_matching_index] - .alternatives_count[matching_index.point_index]; + waypoint.values.emplace("matchings_index", matching_index.sub_matching_index); + waypoint.values.emplace("waypoint_index", matching_index.point_index); + waypoint.values.emplace("alternatives_count", + sub_matchings[matching_index.sub_matching_index] + .alternatives_count[matching_index.point_index]); // waypoint indices need to be adjusted if route legs were collapsed // waypoint parameter assumes there is only one match object if (!parameters.waypoints.empty()) @@ -217,7 +217,7 @@ class MatchAPI final : public RouteAPI waypoint.values["waypoint_index"] = util::json::Null(); } } - waypoints.values.push_back(std::move(waypoint)); + waypoints.values.emplace_back(std::move(waypoint)); } return waypoints; From a0eda3e7d730a0dd3624f8b44b74f6af2ea02f89 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Sat, 15 Jun 2024 18:56:40 +0200 Subject: [PATCH 06/17] Apply micro-optimisation for Route API (#6948) --- CHANGELOG.md | 1 + include/engine/api/route_api.hpp | 70 ++++++++++++++++++-------------- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bd3782cd..4ed316ac8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - NodeJS: - CHANGED: Use node-api instead of NAN. [#6452](https://github.com/Project-OSRM/osrm-backend/pull/6452) - Misc: + - CHANGED: Apply micro-optimisation for Route API. [#6948](https://github.com/Project-OSRM/osrm-backend/pull/6948) - CHANGED: Apply micro-optimisation for Match API. [#6945](https://github.com/Project-OSRM/osrm-backend/pull/6945) - CHANGED: Apply micro-optimisation for Nearest API. [#6944](https://github.com/Project-OSRM/osrm-backend/pull/6944) - CHANGED: Avoid copy of intersection in totalTurnAngle. [#6938](https://github.com/Project-OSRM/osrm-backend/pull/6938) diff --git a/include/engine/api/route_api.hpp b/include/engine/api/route_api.hpp index b6907ddca..211be7f8d 100644 --- a/include/engine/api/route_api.hpp +++ b/include/engine/api/route_api.hpp @@ -110,14 +110,14 @@ class RouteAPI : public BaseAPI if (!parameters.skip_waypoints) { - response.values["waypoints"] = BaseAPI::MakeWaypoints(waypoint_candidates); + response.values.emplace("waypoints", BaseAPI::MakeWaypoints(waypoint_candidates)); } - response.values["routes"] = std::move(jsRoutes); - response.values["code"] = "Ok"; + response.values.emplace("routes", std::move(jsRoutes)); + response.values.emplace("code", "Ok"); auto data_timestamp = facade.GetTimestamp(); if (!data_timestamp.empty()) { - response.values["data_version"] = data_timestamp; + response.values.emplace("data_version", data_timestamp); } } @@ -784,49 +784,57 @@ class RouteAPI : public BaseAPI if (requested_annotations & RouteParameters::AnnotationsType::Speed) { double prev_speed = 0; - annotation.values["speed"] = GetAnnotations( - leg_geometry, - [&prev_speed](const guidance::LegGeometry::Annotation &anno) - { - if (anno.duration < std::numeric_limits::min()) - { - return prev_speed; - } - else - { - auto speed = std::round(anno.distance / anno.duration * 10.) / 10.; - prev_speed = speed; - return util::json::clamp_float(speed); - } - }); + annotation.values.emplace( + "speed", + GetAnnotations(leg_geometry, + [&prev_speed](const guidance::LegGeometry::Annotation &anno) + { + if (anno.duration < std::numeric_limits::min()) + { + return prev_speed; + } + else + { + auto speed = + std::round(anno.distance / anno.duration * 10.) / + 10.; + prev_speed = speed; + return util::json::clamp_float(speed); + } + })); } if (requested_annotations & RouteParameters::AnnotationsType::Duration) { - annotation.values["duration"] = + annotation.values.emplace( + "duration", GetAnnotations(leg_geometry, [](const guidance::LegGeometry::Annotation &anno) - { return anno.duration; }); + { return anno.duration; })); } if (requested_annotations & RouteParameters::AnnotationsType::Distance) { - annotation.values["distance"] = + annotation.values.emplace( + "distance", GetAnnotations(leg_geometry, [](const guidance::LegGeometry::Annotation &anno) - { return anno.distance; }); + { return anno.distance; })); } if (requested_annotations & RouteParameters::AnnotationsType::Weight) { - annotation.values["weight"] = GetAnnotations( - leg_geometry, - [](const guidance::LegGeometry::Annotation &anno) { return anno.weight; }); + annotation.values.emplace( + "weight", + GetAnnotations(leg_geometry, + [](const guidance::LegGeometry::Annotation &anno) + { return anno.weight; })); } if (requested_annotations & RouteParameters::AnnotationsType::Datasources) { - annotation.values["datasources"] = + annotation.values.emplace( + "datasources", GetAnnotations(leg_geometry, [](const guidance::LegGeometry::Annotation &anno) - { return anno.datasource; }); + { return anno.datasource; })); } if (requested_annotations & RouteParameters::AnnotationsType::Nodes) { @@ -837,7 +845,7 @@ class RouteAPI : public BaseAPI nodes.values.push_back( static_cast(facade.GetOSMNodeIDOfNode(node_id))); } - annotation.values["nodes"] = std::move(nodes); + annotation.values.emplace("nodes", std::move(nodes)); } // Add any supporting metadata, if needed if (requested_annotations & RouteParameters::AnnotationsType::Datasources) @@ -853,8 +861,8 @@ class RouteAPI : public BaseAPI break; datasource_names.values.push_back(std::string(facade.GetDatasourceName(i))); } - metadata.values["datasource_names"] = datasource_names; - annotation.values["metadata"] = metadata; + metadata.values.emplace("datasource_names", datasource_names); + annotation.values.emplace("metadata", metadata); } annotations.push_back(std::move(annotation)); From feb938943641e6b34e08cf142b11a1bb7862f56c Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Sat, 15 Jun 2024 18:57:26 +0200 Subject: [PATCH 07/17] Apply micro-optimisation for Table & Trip APIs (#6949) --- CHANGELOG.md | 1 + include/engine/api/table_api.hpp | 27 +++++++++++++++------------ include/engine/api/trip_api.hpp | 12 ++++++------ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ed316ac8..87627e0c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - NodeJS: - CHANGED: Use node-api instead of NAN. [#6452](https://github.com/Project-OSRM/osrm-backend/pull/6452) - Misc: + - CHANGED: Apply micro-optimisation for Table & Trip APIs. [#6949](https://github.com/Project-OSRM/osrm-backend/pull/6949) - CHANGED: Apply micro-optimisation for Route API. [#6948](https://github.com/Project-OSRM/osrm-backend/pull/6948) - CHANGED: Apply micro-optimisation for Match API. [#6945](https://github.com/Project-OSRM/osrm-backend/pull/6945) - CHANGED: Apply micro-optimisation for Nearest API. [#6944](https://github.com/Project-OSRM/osrm-backend/pull/6944) diff --git a/include/engine/api/table_api.hpp b/include/engine/api/table_api.hpp index 10b57f985..82bfe4e38 100644 --- a/include/engine/api/table_api.hpp +++ b/include/engine/api/table_api.hpp @@ -179,7 +179,7 @@ class TableAPI final : public BaseAPI { if (!parameters.skip_waypoints) { - response.values["sources"] = MakeWaypoints(candidates); + response.values.emplace("sources", MakeWaypoints(candidates)); } number_of_sources = candidates.size(); } @@ -187,7 +187,7 @@ class TableAPI final : public BaseAPI { if (!parameters.skip_waypoints) { - response.values["sources"] = MakeWaypoints(candidates, parameters.sources); + response.values.emplace("sources", MakeWaypoints(candidates, parameters.sources)); } } @@ -195,7 +195,7 @@ class TableAPI final : public BaseAPI { if (!parameters.skip_waypoints) { - response.values["destinations"] = MakeWaypoints(candidates); + response.values.emplace("destinations", MakeWaypoints(candidates)); } number_of_destinations = candidates.size(); } @@ -203,34 +203,37 @@ class TableAPI final : public BaseAPI { if (!parameters.skip_waypoints) { - response.values["destinations"] = - MakeWaypoints(candidates, parameters.destinations); + response.values.emplace("destinations", + MakeWaypoints(candidates, parameters.destinations)); } } if (parameters.annotations & TableParameters::AnnotationsType::Duration) { - response.values["durations"] = - MakeDurationTable(tables.first, number_of_sources, number_of_destinations); + response.values.emplace( + "durations", + MakeDurationTable(tables.first, number_of_sources, number_of_destinations)); } if (parameters.annotations & TableParameters::AnnotationsType::Distance) { - response.values["distances"] = - MakeDistanceTable(tables.second, number_of_sources, number_of_destinations); + response.values.emplace( + "distances", + MakeDistanceTable(tables.second, number_of_sources, number_of_destinations)); } if (parameters.fallback_speed != from_alias(INVALID_FALLBACK_SPEED) && parameters.fallback_speed > 0) { - response.values["fallback_speed_cells"] = MakeEstimatesTable(fallback_speed_cells); + response.values.emplace("fallback_speed_cells", + MakeEstimatesTable(fallback_speed_cells)); } - response.values["code"] = "Ok"; + response.values.emplace("code", "Ok"); auto data_timestamp = facade.GetTimestamp(); if (!data_timestamp.empty()) { - response.values["data_version"] = data_timestamp; + response.values.emplace("data_version", data_timestamp); } } diff --git a/include/engine/api/trip_api.hpp b/include/engine/api/trip_api.hpp index 2f821cc1b..96486f026 100644 --- a/include/engine/api/trip_api.hpp +++ b/include/engine/api/trip_api.hpp @@ -79,14 +79,14 @@ class TripAPI final : public RouteAPI } if (!parameters.skip_waypoints) { - response.values["waypoints"] = MakeWaypoints(sub_trips, candidates); + response.values.emplace("waypoints", MakeWaypoints(sub_trips, candidates)); } - response.values["trips"] = std::move(routes); - response.values["code"] = "Ok"; + response.values.emplace("trips", std::move(routes)); + response.values.emplace("code", "Ok"); auto data_timestamp = facade.GetTimestamp(); if (!data_timestamp.empty()) { - response.values["data_version"] = data_timestamp; + response.values.emplace("data_version", data_timestamp); } } @@ -151,8 +151,8 @@ class TripAPI final : public RouteAPI BOOST_ASSERT(!trip_index.NotUsed()); auto waypoint = BaseAPI::MakeWaypoint(candidates[input_index]); - waypoint.values["trips_index"] = trip_index.sub_trip_index; - waypoint.values["waypoint_index"] = trip_index.point_index; + waypoint.values.emplace("trips_index", trip_index.sub_trip_index); + waypoint.values.emplace("waypoint_index", trip_index.point_index); waypoints.values.push_back(std::move(waypoint)); } From 7ffc08be2849742ceb92fbdc756b7af3ba6f8f55 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Wed, 19 Jun 2024 07:46:56 +0200 Subject: [PATCH 08/17] Bump versions of TBB, Expat and Lua installed via Conan (#6957) --- CMakeLists.txt | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1876d6b85..eac368946 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -330,20 +330,12 @@ if(ENABLE_CONAN) set(CONAN_BOOST_VERSION "1.85.0@#14265ec82b25d91305bbb3b30d3357f8") set(CONAN_BZIP2_VERSION "1.0.8@#d1b2d5816f25865acf978501dff1f897") - set(CONAN_EXPAT_VERSION "2.2.10@#916908d4a570ad839edd25322c3268cd") - set(CONAN_LUA_VERSION "5.4.4@#3ec62efc37cd0a5d80b9e5cb35277360") - set(CONAN_TBB_VERSION "2021.3.0@#507ec17cbd51a84167e143b20d170eea") + set(CONAN_EXPAT_VERSION "2.6.2@#2d385d0d50eb5561006a7ff9e356656b") + set(CONAN_LUA_VERSION "5.4.6@#658d6089093cf01992c2737ab2e96763") + set(CONAN_TBB_VERSION "2021.12.0@#e56e5b44be8d690530585dd3634c0106") set(CONAN_SYSTEM_INCLUDES ON) - # TODO: - # if we link TBB dynamically osrm-extract.exe finishes on the first access to any TBB symbol - # with exit code = -1073741515, which means that program cannot load required DLL. - if (MSVC) - set(TBB_SHARED False) - else() - set(TBB_SHARED True) - endif() set(CONAN_ARGS REQUIRES @@ -357,7 +349,6 @@ if(ENABLE_CONAN) KEEP_RPATHS NO_OUTPUT_DIRS OPTIONS boost:filesystem_version=3 # https://stackoverflow.com/questions/73392648/error-with-boost-filesystem-version-in-cmake - onetbb:shared=${TBB_SHARED} boost:without_stacktrace=True # Apple Silicon cross-compilation fails without it BUILD missing ) From 97f676d5a372a8ea6f0e04f8ba4054997a0877fd Mon Sep 17 00:00:00 2001 From: knowname Date: Wed, 19 Jun 2024 14:15:23 +0200 Subject: [PATCH 09/17] add alpine dockerfile (#6958) --- docker/Dockerfile-alpine | 64 ++++++++++++++++++++++++++++++++++++++++ docker/hooks/build | 6 +++- 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 docker/Dockerfile-alpine mode change 100644 => 100755 docker/hooks/build diff --git a/docker/Dockerfile-alpine b/docker/Dockerfile-alpine new file mode 100644 index 000000000..c636b1619 --- /dev/null +++ b/docker/Dockerfile-alpine @@ -0,0 +1,64 @@ +FROM alpine:3.20.0 as alpine-mimalloc + +RUN apk add --no-cache mimalloc + +ENV LD_PRELOAD=/usr/lib/libmimalloc.so.2 +ENV MIMALLOC_LARGE_OS_PAGES=1 + + +FROM alpine-mimalloc as builder +ARG DOCKER_TAG +ARG BUILD_CONCURRENCY +RUN mkdir -p /src && mkdir -p /opt + +RUN apk add --no-cache \ + cmake make git clang libbz2 libxml2 \ + boost-dev boost-program_options boost-filesystem boost-iostreams boost-thread \ + lua5.4-dev onetbb-dev expat-dev + +COPY . /src +WORKDIR /src + +RUN NPROC=${BUILD_CONCURRENCY:-$(nproc)} && \ + echo "Building OSRM ${DOCKER_TAG}" && \ + git show --format="%H" | head -n1 > /opt/OSRM_GITSHA && \ + echo "Building OSRM gitsha $(cat /opt/OSRM_GITSHA)" && \ + mkdir -p build && \ + cd build && \ + BUILD_TYPE="Release" && \ + ENABLE_ASSERTIONS="Off" && \ + BUILD_TOOLS="Off" && \ + case ${DOCKER_TAG} in *"-debug"*) BUILD_TYPE="Debug";; esac && \ + case ${DOCKER_TAG} in *"-assertions"*) BUILD_TYPE="RelWithDebInfo" && ENABLE_ASSERTIONS="On" && BUILD_TOOLS="On";; esac && \ + echo "Building ${BUILD_TYPE} with ENABLE_ASSERTIONS=${ENABLE_ASSERTIONS} BUILD_TOOLS=${BUILD_TOOLS}" && \ + cmake .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DENABLE_ASSERTIONS=${ENABLE_ASSERTIONS} -DBUILD_TOOLS=${BUILD_TOOLS} -DENABLE_LTO=On && \ + make -j${NPROC} install && \ + cd ../profiles && \ + cp -r * /opt && \ + strip /usr/local/bin/* && \ + rm -rf /src + + +# Multistage build to reduce image size - https://docs.docker.com/engine/userguide/eng-image/multistage-build/#use-multi-stage-builds +# Only the content below ends up in the image, this helps remove /src from the image (which is large) +FROM alpine-mimalloc as runstage + +COPY --from=builder /usr/local /usr/local +COPY --from=builder /opt /opt + +RUN apk add --no-cache \ + boost-program_options boost-regex \ + boost-date_time boost-chrono boost-filesystem \ + boost-iostreams boost-system boost-thread \ + expat lua5.4 onetbb && \ + ldconfig /usr/local/lib + +RUN /usr/local/bin/osrm-extract --help && \ + /usr/local/bin/osrm-routed --help && \ + /usr/local/bin/osrm-contract --help && \ + /usr/local/bin/osrm-partition --help && \ + /usr/local/bin/osrm-customize --help + +WORKDIR /opt + +EXPOSE 5000 diff --git a/docker/hooks/build b/docker/hooks/build old mode 100644 new mode 100755 index c27c851ad..8756e3e93 --- a/docker/hooks/build +++ b/docker/hooks/build @@ -6,4 +6,8 @@ # ensure that "COPY . /src" is referring to the repo root, not the directory # that contains the Dockerfile. # This script gets executed with a pwd of wherever the Dockerfile is. -docker build --build-arg BUILD_CONCURRENCY=${CONCURRENCY:-1} --build-arg DOCKER_TAG=${DOCKER_TAG} -t $IMAGE_NAME -f Dockerfile .. + +DOCKER_BUILD="docker build --build-arg BUILD_CONCURRENCY=${CONCURRENCY} --build-arg DOCKER_TAG=${DOCKER_TAG:?unset} -t ${IMAGE_NAME:?unset} -f" + +$DOCKER_BUILD Dockerfile .. +$DOCKER_BUILD Dockerfile-alpine .. From c3f2a6cdb929b2dfd1afa770520dd845473c8de0 Mon Sep 17 00:00:00 2001 From: knowname Date: Thu, 20 Jun 2024 09:08:57 +0200 Subject: [PATCH 10/17] Add CI job for building Alpine-based Docker image (#6959) --- .github/workflows/osrm-backend-docker.yml | 20 +++---- .github/workflows/osrm-backend.yml | 9 ++- docker/Dockerfile | 69 +---------------------- docker/Dockerfile-debian | 68 ++++++++++++++++++++++ 4 files changed, 85 insertions(+), 81 deletions(-) mode change 100644 => 120000 docker/Dockerfile create mode 100644 docker/Dockerfile-debian diff --git a/.github/workflows/osrm-backend-docker.yml b/.github/workflows/osrm-backend-docker.yml index e0cf8939d..499731f90 100644 --- a/.github/workflows/osrm-backend-docker.yml +++ b/.github/workflows/osrm-backend-docker.yml @@ -6,6 +6,9 @@ on: jobs: publish: + strategy: + matrix: + docker-base-image: ["debian", "alpine"] runs-on: ubuntu-latest steps: - name: Check out the repo @@ -53,10 +56,10 @@ jobs: with: push: true platforms: linux/amd64,linux/arm64 - file: ./docker/Dockerfile + file: ./docker/Dockerfile-${{ matrix.docker-base-image }} tags: ${{ steps.metadebug.outputs.tags }} build-args: | - DOCKER_TAG=${{ join(steps.metadebug.outputs.tags ) }} + DOCKER_TAG=${{ join(steps.metadebug.outputs.tags ) }}-${{ matrix.docker-base-image }} - name: Build container image - assertions @@ -64,10 +67,10 @@ jobs: with: push: true platforms: linux/amd64,linux/arm64 - file: ./docker/Dockerfile + file: ./docker/Dockerfile-${{ matrix.docker-base-image }} tags: ${{ steps.metaassertions.outputs.tags }} build-args: | - DOCKER_TAG=${{ join(steps.metaassertions.outputs.tags ) }} + DOCKER_TAG=${{ join(steps.metaassertions.outputs.tags ) }}-${{ matrix.docker-base-image }} # build and publish "normal" image as last to get it listed on top - name: Build container image - normal @@ -75,10 +78,7 @@ jobs: with: push: true platforms: linux/amd64,linux/arm64 - file: ./docker/Dockerfile - tags: ${{ steps.meta.outputs.tags }} + file: ./docker/Dockerfile-${{ matrix.docker-base-image }} + tags: ${{ steps.meta.outputs.tags }} build-args: | - DOCKER_TAG=${{ join(steps.meta.outputs.tags ) }} - - - + DOCKER_TAG=${{ join(steps.meta.outputs.tags ) }}-${{ matrix.docker-base-image }} diff --git a/.github/workflows/osrm-backend.yml b/.github/workflows/osrm-backend.yml index 98c6fbd0f..ca77f6e82 100644 --- a/.github/workflows/osrm-backend.yml +++ b/.github/workflows/osrm-backend.yml @@ -116,7 +116,10 @@ jobs: npm run docs && ./scripts/error_on_dirty.sh npm audit --production - docker-image: + docker-image-matrix: + strategy: + matrix: + docker-base-image: ["debian", "alpine"] needs: format-taginfo-docs runs-on: ubuntu-22.04 continue-on-error: false @@ -132,7 +135,7 @@ jobs: v1-berlin-osm-pbf - name: Docker build run: | - docker build -t osrm-backend-local -f docker/Dockerfile . + docker build -t osrm-backend-local -f docker/Dockerfile-${{ matrix.docker-base-image }} . - name: Test Docker image run: | if [ ! -f "${PWD}/berlin-latest.osm.pbf" ]; then @@ -734,6 +737,6 @@ jobs: ci-complete: runs-on: ubuntu-22.04 - needs: [build-test-publish, docker-image, windows-release-node, benchmarks] + needs: [build-test-publish, docker-image-matrix, windows-release-node, benchmarks] steps: - run: echo "CI complete" diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index ca4ae165d..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,68 +0,0 @@ -FROM debian:bookworm-slim as builder -ARG DOCKER_TAG -ARG BUILD_CONCURRENCY -RUN mkdir -p /src && mkdir -p /opt - -RUN apt-get update && \ - apt-get -y --no-install-recommends install ca-certificates cmake make git gcc g++ libbz2-dev libxml2-dev wget \ - libzip-dev libboost1.81-all-dev lua5.4 liblua5.4-dev pkg-config -o APT::Install-Suggests=0 -o APT::Install-Recommends=0 - -RUN NPROC=${BUILD_CONCURRENCY:-$(nproc)} && \ - ldconfig /usr/local/lib && \ - git clone --branch v2021.12.0 --single-branch https://github.com/oneapi-src/oneTBB.git && \ - cd oneTBB && \ - mkdir build && \ - cd build && \ - cmake -DTBB_TEST=OFF -DCMAKE_BUILD_TYPE=Release .. && \ - cmake --build . && \ - cmake --install . - -COPY . /src -WORKDIR /src - -RUN NPROC=${BUILD_CONCURRENCY:-$(nproc)} && \ - export CXXFLAGS="-Wno-array-bounds -Wno-uninitialized" && \ - echo "Building OSRM ${DOCKER_TAG}" && \ - git show --format="%H" | head -n1 > /opt/OSRM_GITSHA && \ - echo "Building OSRM gitsha $(cat /opt/OSRM_GITSHA)" && \ - mkdir -p build && \ - cd build && \ - BUILD_TYPE="Release" && \ - ENABLE_ASSERTIONS="Off" && \ - BUILD_TOOLS="Off" && \ - case ${DOCKER_TAG} in *"-debug"*) BUILD_TYPE="Debug";; esac && \ - case ${DOCKER_TAG} in *"-assertions"*) BUILD_TYPE="RelWithDebInfo" && ENABLE_ASSERTIONS="On" && BUILD_TOOLS="On";; esac && \ - echo "Building ${BUILD_TYPE} with ENABLE_ASSERTIONS=${ENABLE_ASSERTIONS} BUILD_TOOLS=${BUILD_TOOLS}" && \ - cmake .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DENABLE_ASSERTIONS=${ENABLE_ASSERTIONS} -DBUILD_TOOLS=${BUILD_TOOLS} -DENABLE_LTO=On && \ - make -j${NPROC} install && \ - cd ../profiles && \ - cp -r * /opt && \ - strip /usr/local/bin/* && \ - rm -rf /src - - -# Multistage build to reduce image size - https://docs.docker.com/engine/userguide/eng-image/multistage-build/#use-multi-stage-builds -# Only the content below ends up in the image, this helps remove /src from the image (which is large) -FROM debian:bookworm-slim as runstage - -COPY --from=builder /usr/local /usr/local -COPY --from=builder /opt /opt - -RUN apt-get update && \ - apt-get install -y --no-install-recommends libboost-program-options1.81.0 libboost-regex1.81.0 \ - libboost-date-time1.81.0 libboost-chrono1.81.0 libboost-filesystem1.81.0 \ - libboost-iostreams1.81.0 libboost-system1.81.0 libboost-thread1.81.0 \ - expat liblua5.4-0 && \ - rm -rf /var/lib/apt/lists/* && \ -# add /usr/local/lib to ldconfig to allow loading libraries from there - ldconfig /usr/local/lib - -RUN /usr/local/bin/osrm-extract --help && \ - /usr/local/bin/osrm-routed --help && \ - /usr/local/bin/osrm-contract --help && \ - /usr/local/bin/osrm-partition --help && \ - /usr/local/bin/osrm-customize --help - -WORKDIR /opt - -EXPOSE 5000 diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 120000 index 000000000..4407bedc5 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1 @@ +Dockerfile-debian \ No newline at end of file diff --git a/docker/Dockerfile-debian b/docker/Dockerfile-debian new file mode 100644 index 000000000..ca4ae165d --- /dev/null +++ b/docker/Dockerfile-debian @@ -0,0 +1,68 @@ +FROM debian:bookworm-slim as builder +ARG DOCKER_TAG +ARG BUILD_CONCURRENCY +RUN mkdir -p /src && mkdir -p /opt + +RUN apt-get update && \ + apt-get -y --no-install-recommends install ca-certificates cmake make git gcc g++ libbz2-dev libxml2-dev wget \ + libzip-dev libboost1.81-all-dev lua5.4 liblua5.4-dev pkg-config -o APT::Install-Suggests=0 -o APT::Install-Recommends=0 + +RUN NPROC=${BUILD_CONCURRENCY:-$(nproc)} && \ + ldconfig /usr/local/lib && \ + git clone --branch v2021.12.0 --single-branch https://github.com/oneapi-src/oneTBB.git && \ + cd oneTBB && \ + mkdir build && \ + cd build && \ + cmake -DTBB_TEST=OFF -DCMAKE_BUILD_TYPE=Release .. && \ + cmake --build . && \ + cmake --install . + +COPY . /src +WORKDIR /src + +RUN NPROC=${BUILD_CONCURRENCY:-$(nproc)} && \ + export CXXFLAGS="-Wno-array-bounds -Wno-uninitialized" && \ + echo "Building OSRM ${DOCKER_TAG}" && \ + git show --format="%H" | head -n1 > /opt/OSRM_GITSHA && \ + echo "Building OSRM gitsha $(cat /opt/OSRM_GITSHA)" && \ + mkdir -p build && \ + cd build && \ + BUILD_TYPE="Release" && \ + ENABLE_ASSERTIONS="Off" && \ + BUILD_TOOLS="Off" && \ + case ${DOCKER_TAG} in *"-debug"*) BUILD_TYPE="Debug";; esac && \ + case ${DOCKER_TAG} in *"-assertions"*) BUILD_TYPE="RelWithDebInfo" && ENABLE_ASSERTIONS="On" && BUILD_TOOLS="On";; esac && \ + echo "Building ${BUILD_TYPE} with ENABLE_ASSERTIONS=${ENABLE_ASSERTIONS} BUILD_TOOLS=${BUILD_TOOLS}" && \ + cmake .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DENABLE_ASSERTIONS=${ENABLE_ASSERTIONS} -DBUILD_TOOLS=${BUILD_TOOLS} -DENABLE_LTO=On && \ + make -j${NPROC} install && \ + cd ../profiles && \ + cp -r * /opt && \ + strip /usr/local/bin/* && \ + rm -rf /src + + +# Multistage build to reduce image size - https://docs.docker.com/engine/userguide/eng-image/multistage-build/#use-multi-stage-builds +# Only the content below ends up in the image, this helps remove /src from the image (which is large) +FROM debian:bookworm-slim as runstage + +COPY --from=builder /usr/local /usr/local +COPY --from=builder /opt /opt + +RUN apt-get update && \ + apt-get install -y --no-install-recommends libboost-program-options1.81.0 libboost-regex1.81.0 \ + libboost-date-time1.81.0 libboost-chrono1.81.0 libboost-filesystem1.81.0 \ + libboost-iostreams1.81.0 libboost-system1.81.0 libboost-thread1.81.0 \ + expat liblua5.4-0 && \ + rm -rf /var/lib/apt/lists/* && \ +# add /usr/local/lib to ldconfig to allow loading libraries from there + ldconfig /usr/local/lib + +RUN /usr/local/bin/osrm-extract --help && \ + /usr/local/bin/osrm-routed --help && \ + /usr/local/bin/osrm-contract --help && \ + /usr/local/bin/osrm-partition --help && \ + /usr/local/bin/osrm-customize --help + +WORKDIR /opt + +EXPOSE 5000 From f1eabc3ecc235fcaec41c304915ce35f5740c1a6 Mon Sep 17 00:00:00 2001 From: Dennis Luxen Date: Thu, 20 Jun 2024 21:44:28 +0200 Subject: [PATCH 11/17] Replace boost::filesystem with std (#6432) --- .github/workflows/osrm-backend.yml | 2 +- CHANGELOG.md | 1 + features/support/env.js | 2 +- include/contractor/contractor_config.hpp | 5 +- include/contractor/files.hpp | 4 +- include/customizer/customizer_config.hpp | 5 +- include/customizer/edge_based_graph.hpp | 2 +- include/customizer/files.hpp | 8 +- .../contiguous_internalmem_datafacade.hpp | 2 +- include/engine/engine_config.hpp | 5 +- include/extractor/extractor_config.hpp | 13 +-- include/extractor/files.hpp | 110 +++++++++--------- include/extractor/location_dependent_data.hpp | 6 +- .../extractor/node_based_graph_factory.hpp | 3 +- include/extractor/raster_source.hpp | 5 +- .../extractor/scripting_environment_lua.hpp | 2 +- include/extractor/segment_data_container.hpp | 5 +- include/guidance/files.hpp | 8 +- .../partitioner/edge_based_graph_reader.hpp | 6 +- include/partitioner/files.hpp | 12 +- include/partitioner/partitioner_config.hpp | 5 +- include/storage/io.hpp | 27 +++-- include/storage/io_config.hpp | 26 ++--- include/storage/shared_memory.hpp | 18 +-- include/storage/storage.hpp | 11 +- include/storage/storage_config.hpp | 13 ++- include/storage/tar.hpp | 12 +- include/storage/view_factory.hpp | 2 +- include/updater/csv_file_parser.hpp | 4 +- include/updater/updater_config.hpp | 11 +- include/util/lua_util.hpp | 4 +- include/util/mmap_file.hpp | 19 +-- include/util/mmap_tar.hpp | 2 +- include/util/static_rtree.hpp | 8 +- include/util/timezones.hpp | 4 +- scripts/ci/windows-build.bat | 96 +++++++++++++++ src/benchmarks/static_rtree.cpp | 3 +- .../datafacade/mmap_memory_allocator.cpp | 2 +- src/extractor/compressed_edge_container.cpp | 5 +- src/extractor/location_dependent_data.cpp | 8 +- src/extractor/raster_source.cpp | 4 +- src/extractor/scripting_environment_lua.cpp | 2 +- src/partitioner/partitioner.cpp | 15 ++- src/storage/io_config.cpp | 6 +- src/storage/storage.cpp | 50 ++++---- src/tools/components.cpp | 4 +- src/tools/contract.cpp | 6 +- src/tools/customize.cpp | 6 +- src/tools/extract.cpp | 18 ++- src/tools/io-benchmark.cpp | 17 ++- src/tools/partition.cpp | 10 +- src/tools/routed.cpp | 14 +-- src/tools/store.cpp | 10 +- src/updater/updater.cpp | 4 +- src/util/timezones.cpp | 6 +- unit_tests/common/temporary_file.hpp | 31 ++++- .../location_dependent_data_tests.cpp | 20 ++-- unit_tests/extractor/name_table.cpp | 2 +- unit_tests/extractor/raster_source.cpp | 3 +- unit_tests/storage/data_layout.cpp | 2 +- unit_tests/storage/serialization.cpp | 2 +- unit_tests/updater/timezoner.cpp | 5 +- unit_tests/util/serialization.cpp | 3 +- unit_tests/util/static_rtree.cpp | 2 +- 64 files changed, 417 insertions(+), 311 deletions(-) create mode 100644 scripts/ci/windows-build.bat diff --git a/.github/workflows/osrm-backend.yml b/.github/workflows/osrm-backend.yml index ca77f6e82..ee8af27b9 100644 --- a/.github/workflows/osrm-backend.yml +++ b/.github/workflows/osrm-backend.yml @@ -152,7 +152,6 @@ jobs: >&2 echo "No berlin-latest.geojson found" exit 1 fi - # removing `.osrm.nbg` to check that whole pipeline works without it rm -rf "${PWD}/berlin-latest.osrm.nbg" @@ -740,3 +739,4 @@ jobs: needs: [build-test-publish, docker-image-matrix, windows-release-node, benchmarks] steps: - run: echo "CI complete" + diff --git a/CHANGELOG.md b/CHANGELOG.md index 87627e0c9..e5b4c9662 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ - CHANGED: Avoid copy of std::function-based callback in path unpacking [#6895](https://github.com/Project-OSRM/osrm-backend/pull/6895) - CHANGED: Replace boost::hash by std::hash [#6892](https://github.com/Project-OSRM/osrm-backend/pull/6892) - CHANGED: Partial fix migration from boost::optional to std::optional [#6551](https://github.com/Project-OSRM/osrm-backend/issues/6551) + - CHANGED: Replace boost::filesystem with std::filesystem [#6432](https://github.com/Project-OSRM/osrm-backend/pull/6432) - CHANGED: Update Conan Boost version to 1.85.0. [#6868](https://github.com/Project-OSRM/osrm-backend/pull/6868) - FIXED: Fix an error in a RouteParameters AnnotationsType operator overload. [#6646](https://github.com/Project-OSRM/osrm-backend/pull/6646) - ADDED: Add support for "unlimited" to be passed as a value for the default-radius and max-matching-radius flags. [#6599](https://github.com/Project-OSRM/osrm-backend/pull/6599) diff --git a/features/support/env.js b/features/support/env.js index 85b7b4383..7914e3165 100644 --- a/features/support/env.js +++ b/features/support/env.js @@ -42,7 +42,7 @@ module.exports = function () { this.OSRM_PORT = process.env.OSRM_PORT && parseInt(process.env.OSRM_PORT) || 5000; this.OSRM_IP = process.env.OSRM_IP || '127.0.0.1'; this.OSRM_CONNECTION_RETRIES = process.env.OSRM_CONNECTION_RETRIES && parseInt(process.env.OSRM_CONNECTION_RETRIES) || 10; - this.OSRM_CONNECTION_EXP_BACKOFF_COEF = process.env.OSRM_CONNECTION_EXP_BACKOFF_COEF && parseFloat(process.env.OSRM_CONNECTION_EXP_BACKOFF_COEF) || 1.0; + this.OSRM_CONNECTION_EXP_BACKOFF_COEF = process.env.OSRM_CONNECTION_EXP_BACKOFF_COEF && parseFloat(process.env.OSRM_CONNECTION_EXP_BACKOFF_COEF) || 1.1; this.HOST = `http://${this.OSRM_IP}:${this.OSRM_PORT}`; diff --git a/include/contractor/contractor_config.hpp b/include/contractor/contractor_config.hpp index 020720dde..30889e218 100644 --- a/include/contractor/contractor_config.hpp +++ b/include/contractor/contractor_config.hpp @@ -31,8 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "storage/io_config.hpp" #include "updater/updater_config.hpp" -#include - +#include #include namespace osrm::contractor @@ -47,7 +46,7 @@ struct ContractorConfig final : storage::IOConfig } // Infer the output names from the path of the .osrm file - void UseDefaultOutputNames(const boost::filesystem::path &base) + void UseDefaultOutputNames(const std::filesystem::path &base) { IOConfig::UseDefaultOutputNames(base); updater_config.UseDefaultOutputNames(base); diff --git a/include/contractor/files.hpp b/include/contractor/files.hpp index c0755b728..7b77869bf 100644 --- a/include/contractor/files.hpp +++ b/include/contractor/files.hpp @@ -9,7 +9,7 @@ namespace osrm::contractor::files { // reads .osrm.hsgr file template -inline void readGraph(const boost::filesystem::path &path, +inline void readGraph(const std::filesystem::path &path, std::unordered_map &metrics, std::uint32_t &connectivity_checksum) { @@ -30,7 +30,7 @@ inline void readGraph(const boost::filesystem::path &path, // writes .osrm.hsgr file template -inline void writeGraph(const boost::filesystem::path &path, +inline void writeGraph(const std::filesystem::path &path, const std::unordered_map &metrics, const std::uint32_t connectivity_checksum) { diff --git a/include/customizer/customizer_config.hpp b/include/customizer/customizer_config.hpp index 3ac028027..11f786c46 100644 --- a/include/customizer/customizer_config.hpp +++ b/include/customizer/customizer_config.hpp @@ -1,9 +1,8 @@ #ifndef OSRM_CUSTOMIZE_CUSTOMIZER_CONFIG_HPP #define OSRM_CUSTOMIZE_CUSTOMIZER_CONFIG_HPP -#include - #include +#include #include #include "storage/io_config.hpp" @@ -27,7 +26,7 @@ struct CustomizationConfig final : storage::IOConfig { } - void UseDefaultOutputNames(const boost::filesystem::path &base) + void UseDefaultOutputNames(const std::filesystem::path &base) { IOConfig::UseDefaultOutputNames(base); updater_config.UseDefaultOutputNames(base); diff --git a/include/customizer/edge_based_graph.hpp b/include/customizer/edge_based_graph.hpp index 5bc45e24a..9cf92212a 100644 --- a/include/customizer/edge_based_graph.hpp +++ b/include/customizer/edge_based_graph.hpp @@ -9,7 +9,7 @@ #include "storage/shared_memory_ownership.hpp" -#include +#include namespace osrm::customizer { diff --git a/include/customizer/files.hpp b/include/customizer/files.hpp index e5394da59..9a818d534 100644 --- a/include/customizer/files.hpp +++ b/include/customizer/files.hpp @@ -14,7 +14,7 @@ namespace osrm::customizer::files // reads .osrm.cell_metrics file template -inline void readCellMetrics(const boost::filesystem::path &path, +inline void readCellMetrics(const std::filesystem::path &path, std::unordered_map> &metrics) { static_assert(std::is_same::value || @@ -44,7 +44,7 @@ inline void readCellMetrics(const boost::filesystem::path &path, // writes .osrm.cell_metrics file template inline void -writeCellMetrics(const boost::filesystem::path &path, +writeCellMetrics(const std::filesystem::path &path, const std::unordered_map> &metrics) { static_assert(std::is_same::value || @@ -72,7 +72,7 @@ writeCellMetrics(const boost::filesystem::path &path, // reads .osrm.mldgr file template -inline void readGraph(const boost::filesystem::path &path, +inline void readGraph(const std::filesystem::path &path, MultiLevelGraphT &graph, std::uint32_t &connectivity_checksum) { @@ -88,7 +88,7 @@ inline void readGraph(const boost::filesystem::path &path, // writes .osrm.mldgr file template -inline void writeGraph(const boost::filesystem::path &path, +inline void writeGraph(const std::filesystem::path &path, const MultiLevelGraphT &graph, const std::uint32_t connectivity_checksum) { diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp index b6e1532d3..7a495e86f 100644 --- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp +++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp @@ -177,7 +177,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade SharedRTree m_static_rtree; std::unique_ptr m_geospatial_query; - boost::filesystem::path file_index_path; + std::filesystem::path file_index_path; std::optional intersection_bearings_view; diff --git a/include/engine/engine_config.hpp b/include/engine/engine_config.hpp index cb986be9e..439ec8aa6 100644 --- a/include/engine/engine_config.hpp +++ b/include/engine/engine_config.hpp @@ -31,8 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "storage/storage_config.hpp" #include "osrm/datasets.hpp" -#include - +#include #include #include @@ -83,7 +82,7 @@ struct EngineConfig final boost::optional default_radius = -1.0; int max_alternatives = 3; // set an arbitrary upper bound; can be adjusted by user bool use_shared_memory = true; - boost::filesystem::path memory_file; + std::filesystem::path memory_file; bool use_mmap = true; Algorithm algorithm = Algorithm::CH; std::vector disable_feature_dataset; diff --git a/include/extractor/extractor_config.hpp b/include/extractor/extractor_config.hpp index 61a69d6d0..15df396dc 100644 --- a/include/extractor/extractor_config.hpp +++ b/include/extractor/extractor_config.hpp @@ -28,13 +28,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef EXTRACTOR_CONFIG_HPP #define EXTRACTOR_CONFIG_HPP -#include +#include "storage/io_config.hpp" #include +#include #include -#include "storage/io_config.hpp" - namespace osrm::extractor { @@ -71,14 +70,14 @@ struct ExtractorConfig final : storage::IOConfig { } - void UseDefaultOutputNames(const boost::filesystem::path &base) + void UseDefaultOutputNames(const std::filesystem::path &base) { IOConfig::UseDefaultOutputNames(base); } - boost::filesystem::path input_path; - boost::filesystem::path profile_path; - std::vector location_dependent_data_paths; + std::filesystem::path input_path; + std::filesystem::path profile_path; + std::vector location_dependent_data_paths; std::string data_version; unsigned requested_num_threads = 0; diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index bb4a4a2bd..2c90d3a04 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -23,9 +23,9 @@ namespace osrm::extractor::files // writes the .osrm.icd file template -inline void writeIntersections(const boost::filesystem::path &path, - const IntersectionBearingsT &intersection_bearings, - const EntryClassVectorT &entry_classes) +void writeIntersections(const std::filesystem::path &path, + const IntersectionBearingsT &intersection_bearings, + const EntryClassVectorT &entry_classes) { static_assert(std::is_same::value || std::is_same::value, @@ -39,9 +39,9 @@ inline void writeIntersections(const boost::filesystem::path &path, // read the .osrm.icd file template -inline void readIntersections(const boost::filesystem::path &path, - IntersectionBearingsT &intersection_bearings, - EntryClassVectorT &entry_classes) +void readIntersections(const std::filesystem::path &path, + IntersectionBearingsT &intersection_bearings, + EntryClassVectorT &entry_classes) { static_assert(std::is_same::value || std::is_same::value, @@ -54,8 +54,7 @@ inline void readIntersections(const boost::filesystem::path &path, } // reads .osrm.properties -inline void readProfileProperties(const boost::filesystem::path &path, - ProfileProperties &properties) +inline void readProfileProperties(const std::filesystem::path &path, ProfileProperties &properties) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; @@ -64,7 +63,7 @@ inline void readProfileProperties(const boost::filesystem::path &path, } // writes .osrm.properties -inline void writeProfileProperties(const boost::filesystem::path &path, +inline void writeProfileProperties(const std::filesystem::path &path, const ProfileProperties &properties) { const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; @@ -74,7 +73,7 @@ inline void writeProfileProperties(const boost::filesystem::path &path, } template -void writeEdgeBasedGraph(const boost::filesystem::path &path, +void writeEdgeBasedGraph(const std::filesystem::path &path, EdgeID const number_of_edge_based_nodes, const EdgeBasedEdgeVector &edge_based_edge_list, const std::uint32_t connectivity_checksum) @@ -92,7 +91,7 @@ void writeEdgeBasedGraph(const boost::filesystem::path &path, // reads .osrm.ebg file template -void readEdgeBasedGraph(const boost::filesystem::path &path, +void readEdgeBasedGraph(const std::filesystem::path &path, EdgeID &number_of_edge_based_nodes, EdgeBasedEdgeVector &edge_based_edge_list, std::uint32_t &connectivity_checksum) @@ -108,9 +107,9 @@ void readEdgeBasedGraph(const boost::filesystem::path &path, // reads .osrm.nbg_nodes template -inline void readNodes(const boost::filesystem::path &path, - CoordinatesT &coordinates, - PackedOSMIDsT &osm_node_ids) +void readNodes(const std::filesystem::path &path, + CoordinatesT &coordinates, + PackedOSMIDsT &osm_node_ids) { static_assert(std::is_same::value, ""); static_assert(std::is_same::value, ""); @@ -124,7 +123,7 @@ inline void readNodes(const boost::filesystem::path &path, // reads only coordinates from .osrm.nbg_nodes template -inline void readNodeCoordinates(const boost::filesystem::path &path, CoordinatesT &coordinates) +void readNodeCoordinates(const std::filesystem::path &path, CoordinatesT &coordinates) { static_assert(std::is_same::value, ""); @@ -136,9 +135,9 @@ inline void readNodeCoordinates(const boost::filesystem::path &path, Coordinates // writes .osrm.nbg_nodes template -inline void writeNodes(const boost::filesystem::path &path, - const CoordinatesT &coordinates, - const PackedOSMIDsT &osm_node_ids) +void writeNodes(const std::filesystem::path &path, + const CoordinatesT &coordinates, + const PackedOSMIDsT &osm_node_ids) { static_assert(std::is_same::value, ""); static_assert(std::is_same::value, ""); @@ -151,7 +150,7 @@ inline void writeNodes(const boost::filesystem::path &path, } // reads .osrm.cnbg_to_ebg -inline void readNBGMapping(const boost::filesystem::path &path, std::vector &mapping) +inline void readNBGMapping(const std::filesystem::path &path, std::vector &mapping) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; @@ -160,8 +159,7 @@ inline void readNBGMapping(const boost::filesystem::path &path, std::vector &mapping) +inline void writeNBGMapping(const std::filesystem::path &path, const std::vector &mapping) { const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; storage::tar::FileWriter writer{path, fingerprint}; @@ -170,7 +168,7 @@ inline void writeNBGMapping(const boost::filesystem::path &path, } // reads .osrm.datasource_names -inline void readDatasources(const boost::filesystem::path &path, Datasources &sources) +inline void readDatasources(const std::filesystem::path &path, Datasources &sources) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; @@ -179,7 +177,7 @@ inline void readDatasources(const boost::filesystem::path &path, Datasources &so } // writes .osrm.datasource_names -inline void writeDatasources(const boost::filesystem::path &path, Datasources &sources) +inline void writeDatasources(const std::filesystem::path &path, Datasources &sources) { const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; storage::tar::FileWriter writer{path, fingerprint}; @@ -189,7 +187,7 @@ inline void writeDatasources(const boost::filesystem::path &path, Datasources &s // reads .osrm.geometry template -inline void readSegmentData(const boost::filesystem::path &path, SegmentDataT &segment_data) +void readSegmentData(const std::filesystem::path &path, SegmentDataT &segment_data) { static_assert(std::is_same::value || std::is_same::value, @@ -202,7 +200,7 @@ inline void readSegmentData(const boost::filesystem::path &path, SegmentDataT &s // writes .osrm.geometry template -inline void writeSegmentData(const boost::filesystem::path &path, const SegmentDataT &segment_data) +void writeSegmentData(const std::filesystem::path &path, const SegmentDataT &segment_data) { static_assert(std::is_same::value || std::is_same::value, @@ -215,7 +213,7 @@ inline void writeSegmentData(const boost::filesystem::path &path, const SegmentD // reads .osrm.ebg_nodes template -inline void readNodeData(const boost::filesystem::path &path, NodeDataT &node_data) +inline void readNodeData(const std::filesystem::path &path, NodeDataT &node_data) { static_assert(std::is_same::value || std::is_same::value || @@ -229,7 +227,7 @@ inline void readNodeData(const boost::filesystem::path &path, NodeDataT &node_da // writes .osrm.ebg_nodes template -inline void writeNodeData(const boost::filesystem::path &path, const NodeDataT &node_data) +inline void writeNodeData(const std::filesystem::path &path, const NodeDataT &node_data) { static_assert(std::is_same::value || std::is_same::value || @@ -243,7 +241,7 @@ inline void writeNodeData(const boost::filesystem::path &path, const NodeDataT & // reads .osrm.tls template -inline void readTurnLaneDescriptions(const boost::filesystem::path &path, +inline void readTurnLaneDescriptions(const std::filesystem::path &path, OffsetsT &turn_offsets, MaskT &turn_masks) { @@ -260,7 +258,7 @@ inline void readTurnLaneDescriptions(const boost::filesystem::path &path, // writes .osrm.tls template -inline void writeTurnLaneDescriptions(const boost::filesystem::path &path, +inline void writeTurnLaneDescriptions(const std::filesystem::path &path, const OffsetsT &turn_offsets, const MaskT &turn_masks) { @@ -277,7 +275,7 @@ inline void writeTurnLaneDescriptions(const boost::filesystem::path &path, // reads .osrm.tld template -inline void readTurnLaneData(const boost::filesystem::path &path, TurnLaneDataT &turn_lane_data) +inline void readTurnLaneData(const std::filesystem::path &path, TurnLaneDataT &turn_lane_data) { static_assert( std::is_same::value, @@ -291,7 +289,7 @@ inline void readTurnLaneData(const boost::filesystem::path &path, TurnLaneDataT // writes .osrm.tld template -inline void writeTurnLaneData(const boost::filesystem::path &path, +inline void writeTurnLaneData(const std::filesystem::path &path, const TurnLaneDataT &turn_lane_data) { static_assert( @@ -306,7 +304,7 @@ inline void writeTurnLaneData(const boost::filesystem::path &path, // reads .osrm.timestamp template -inline void readTimestamp(const boost::filesystem::path &path, TimestampDataT ×tamp) +inline void readTimestamp(const std::filesystem::path &path, TimestampDataT ×tamp) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; @@ -316,7 +314,7 @@ inline void readTimestamp(const boost::filesystem::path &path, TimestampDataT &t // writes .osrm.timestamp template -inline void writeTimestamp(const boost::filesystem::path &path, const TimestampDataT ×tamp) +inline void writeTimestamp(const std::filesystem::path &path, const TimestampDataT ×tamp) { const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; storage::tar::FileWriter writer{path, fingerprint}; @@ -326,7 +324,7 @@ inline void writeTimestamp(const boost::filesystem::path &path, const TimestampD // reads .osrm.maneuver_overrides template -inline void readManeuverOverrides(const boost::filesystem::path &path, +inline void readManeuverOverrides(const std::filesystem::path &path, StorageManeuverOverrideT &maneuver_overrides, NodeSequencesT &node_sequences) { @@ -340,7 +338,7 @@ inline void readManeuverOverrides(const boost::filesystem::path &path, } // writes .osrm.maneuver_overrides -inline void writeManeuverOverrides(const boost::filesystem::path &path, +inline void writeManeuverOverrides(const std::filesystem::path &path, const std::vector &maneuver_overrides, const std::vector &node_sequences) { @@ -355,7 +353,7 @@ inline void writeManeuverOverrides(const boost::filesystem::path &path, // writes .osrm.turn_weight_penalties template -inline void writeTurnWeightPenalty(const boost::filesystem::path &path, +inline void writeTurnWeightPenalty(const std::filesystem::path &path, const TurnPenaltyT &turn_penalty) { const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; @@ -366,7 +364,7 @@ inline void writeTurnWeightPenalty(const boost::filesystem::path &path, // read .osrm.turn_weight_penalties template -inline void readTurnWeightPenalty(const boost::filesystem::path &path, TurnPenaltyT &turn_penalty) +inline void readTurnWeightPenalty(const std::filesystem::path &path, TurnPenaltyT &turn_penalty) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; @@ -376,7 +374,7 @@ inline void readTurnWeightPenalty(const boost::filesystem::path &path, TurnPenal // writes .osrm.turn_duration_penalties template -inline void writeTurnDurationPenalty(const boost::filesystem::path &path, +inline void writeTurnDurationPenalty(const std::filesystem::path &path, const TurnPenaltyT &turn_penalty) { const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; @@ -387,7 +385,7 @@ inline void writeTurnDurationPenalty(const boost::filesystem::path &path, // read .osrm.turn_weight_penalties template -inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPenaltyT &turn_penalty) +inline void readTurnDurationPenalty(const std::filesystem::path &path, TurnPenaltyT &turn_penalty) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; @@ -397,7 +395,7 @@ inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPen // writes .osrm.turn_penalties_index template -inline void writeTurnPenaltiesIndex(const boost::filesystem::path &path, +inline void writeTurnPenaltiesIndex(const std::filesystem::path &path, const TurnIndexT &turn_penalties_index) { const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; @@ -408,7 +406,7 @@ inline void writeTurnPenaltiesIndex(const boost::filesystem::path &path, // read .osrm.turn_penalties_index template -inline void readTurnPenaltiesIndex(const boost::filesystem::path &path, +inline void readTurnPenaltiesIndex(const std::filesystem::path &path, TurnIndexT &turn_penalties_index) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; @@ -419,7 +417,7 @@ inline void readTurnPenaltiesIndex(const boost::filesystem::path &path, // writes .osrm.restrictions template -inline void writeConditionalRestrictions(const boost::filesystem::path &path, +inline void writeConditionalRestrictions(const std::filesystem::path &path, const ConditionalRestrictionsT &conditional_restrictions) { const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; @@ -430,7 +428,7 @@ inline void writeConditionalRestrictions(const boost::filesystem::path &path, // read .osrm.restrictions template -inline void readConditionalRestrictions(const boost::filesystem::path &path, +inline void readConditionalRestrictions(const std::filesystem::path &path, ConditionalRestrictionsT &conditional_restrictions) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; @@ -441,7 +439,7 @@ inline void readConditionalRestrictions(const boost::filesystem::path &path, // reads .osrm file which is a temporary file of osrm-extract template -void readRawNBGraph(const boost::filesystem::path &path, +void readRawNBGraph(const std::filesystem::path &path, std::vector &coordinates, PackedOSMIDsT &osm_node_ids, std::vector &edge_list) @@ -466,8 +464,7 @@ void readRawNBGraph(const boost::filesystem::path &path, storage::serialization::read(reader, "/extractor/edges", edge_list); } -template -void readNames(const boost::filesystem::path &path, NameTableT &table) +template void readNames(const std::filesystem::path &path, NameTableT &table) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; @@ -476,7 +473,7 @@ void readNames(const boost::filesystem::path &path, NameTableT &table) } template -void writeNames(const boost::filesystem::path &path, const NameTableT &table) +void writeNames(const std::filesystem::path &path, const NameTableT &table) { const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; storage::tar::FileWriter writer{path, fingerprint}; @@ -485,7 +482,7 @@ void writeNames(const boost::filesystem::path &path, const NameTableT &table) } template -void readEdgeBasedNodeWeights(const boost::filesystem::path &path, NodeWeightsVectorT &weights) +void readEdgeBasedNodeWeights(const std::filesystem::path &path, NodeWeightsVectorT &weights) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; @@ -494,8 +491,7 @@ void readEdgeBasedNodeWeights(const boost::filesystem::path &path, NodeWeightsVe } template -void readEdgeBasedNodeDistances(const boost::filesystem::path &path, - NodeDistancesVectorT &distances) +void readEdgeBasedNodeDistances(const std::filesystem::path &path, NodeDistancesVectorT &distances) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; @@ -504,7 +500,7 @@ void readEdgeBasedNodeDistances(const boost::filesystem::path &path, } template -void writeEdgeBasedNodeWeightsDurationsDistances(const boost::filesystem::path &path, +void writeEdgeBasedNodeWeightsDurationsDistances(const std::filesystem::path &path, const NodeWeightsVectorT &weights, const NodeDurationsVectorT &durations, const NodeDistancesVectorT &distances) @@ -518,7 +514,7 @@ void writeEdgeBasedNodeWeightsDurationsDistances(const boost::filesystem::path & } template -void readEdgeBasedNodeWeightsDurations(const boost::filesystem::path &path, +void readEdgeBasedNodeWeightsDurations(const std::filesystem::path &path, NodeWeightsVectorT &weights, NodeDurationsVectorT &durations) { @@ -530,7 +526,7 @@ void readEdgeBasedNodeWeightsDurations(const boost::filesystem::path &path, } template -void writeEdgeBasedNodeWeightsDurations(const boost::filesystem::path &path, +void writeEdgeBasedNodeWeightsDurations(const std::filesystem::path &path, const NodeWeightsVectorT &weights, const NodeDurationsVectorT &durations) { @@ -542,7 +538,7 @@ void writeEdgeBasedNodeWeightsDurations(const boost::filesystem::path &path, } template -void writeRamIndex(const boost::filesystem::path &path, const RTreeT &rtree) +void writeRamIndex(const std::filesystem::path &path, const RTreeT &rtree) { const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; storage::tar::FileWriter writer{path, fingerprint}; @@ -550,7 +546,7 @@ void writeRamIndex(const boost::filesystem::path &path, const RTreeT &rtree) util::serialization::write(writer, "/common/rtree", rtree); } -template void readRamIndex(const boost::filesystem::path &path, RTreeT &rtree) +template void readRamIndex(const std::filesystem::path &path, RTreeT &rtree) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; @@ -559,7 +555,7 @@ template void readRamIndex(const boost::filesystem::path &path } template -void writeCompressedNodeBasedGraph(const boost::filesystem::path &path, const EdgeListT &edge_list) +void writeCompressedNodeBasedGraph(const std::filesystem::path &path, const EdgeListT &edge_list) { const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; storage::tar::FileWriter writer{path, fingerprint}; @@ -568,7 +564,7 @@ void writeCompressedNodeBasedGraph(const boost::filesystem::path &path, const Ed } template -void readCompressedNodeBasedGraph(const boost::filesystem::path &path, EdgeListT &edge_list) +void readCompressedNodeBasedGraph(const std::filesystem::path &path, EdgeListT &edge_list) { const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; diff --git a/include/extractor/location_dependent_data.hpp b/include/extractor/location_dependent_data.hpp index 3c9fd6df3..865ddc5b6 100644 --- a/include/extractor/location_dependent_data.hpp +++ b/include/extractor/location_dependent_data.hpp @@ -1,13 +1,13 @@ #ifndef OSRM_LOCATION_DEPENDENT_DATA_HPP #define OSRM_LOCATION_DEPENDENT_DATA_HPP -#include #include #include #include #include +#include #include #include @@ -30,7 +30,7 @@ struct LocationDependentData using property_t = boost::variant; using properties_t = std::unordered_map; - LocationDependentData(const std::vector &file_paths); + LocationDependentData(const std::vector &file_paths); bool empty() const { return rtree.empty(); } @@ -39,7 +39,7 @@ struct LocationDependentData property_t FindByKey(const std::vector &property_indexes, const char *key) const; private: - void loadLocationDependentData(const boost::filesystem::path &file_path, + void loadLocationDependentData(const std::filesystem::path &file_path, std::vector &bounding_boxes); rtree_t rtree; diff --git a/include/extractor/node_based_graph_factory.hpp b/include/extractor/node_based_graph_factory.hpp index c69842d3c..e3a507285 100644 --- a/include/extractor/node_based_graph_factory.hpp +++ b/include/extractor/node_based_graph_factory.hpp @@ -12,8 +12,7 @@ #include "util/coordinate.hpp" #include "util/node_based_graph.hpp" -#include - +#include #include #include #include diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp index b575b5581..89d438183 100644 --- a/include/extractor/raster_source.hpp +++ b/include/extractor/raster_source.hpp @@ -7,14 +7,13 @@ #include #include #include -#include -#include #include #include #include #include +#include #include #include #include @@ -40,7 +39,7 @@ struct RasterDatum class RasterGrid { public: - RasterGrid(const boost::filesystem::path &filepath, std::size_t _xdim, std::size_t _ydim) + RasterGrid(const std::filesystem::path &filepath, std::size_t _xdim, std::size_t _ydim) { xdim = _xdim; ydim = _ydim; diff --git a/include/extractor/scripting_environment_lua.hpp b/include/extractor/scripting_environment_lua.hpp index ef5243454..26da7a3b5 100644 --- a/include/extractor/scripting_environment_lua.hpp +++ b/include/extractor/scripting_environment_lua.hpp @@ -70,7 +70,7 @@ class Sol2ScriptingEnvironment final : public ScriptingEnvironment explicit Sol2ScriptingEnvironment( const std::string &file_name, - const std::vector &location_dependent_data_paths); + const std::vector &location_dependent_data_paths); ~Sol2ScriptingEnvironment() override = default; const ProfileProperties &GetProfileProperties() override; diff --git a/include/extractor/segment_data_container.hpp b/include/extractor/segment_data_container.hpp index 99bf5d74b..abc5b7a17 100644 --- a/include/extractor/segment_data_container.hpp +++ b/include/extractor/segment_data_container.hpp @@ -8,13 +8,12 @@ #include "storage/shared_memory_ownership.hpp" #include "storage/tar_fwd.hpp" -#include #include #include -#include - +#include #include +#include #include namespace osrm::extractor diff --git a/include/guidance/files.hpp b/include/guidance/files.hpp index b1b7a9776..e72a9f622 100644 --- a/include/guidance/files.hpp +++ b/include/guidance/files.hpp @@ -10,12 +10,16 @@ #include +#include +#include +#include + namespace osrm::guidance::files { // reads .osrm.edges template -inline void readTurnData(const boost::filesystem::path &path, +inline void readTurnData(const std::filesystem::path &path, TurnDataT &turn_data, std::uint32_t &connectivity_checksum) { @@ -32,7 +36,7 @@ inline void readTurnData(const boost::filesystem::path &path, // writes .osrm.edges template -inline void writeTurnData(const boost::filesystem::path &path, +inline void writeTurnData(const std::filesystem::path &path, const TurnDataT &turn_data, const std::uint32_t connectivity_checksum) { diff --git a/include/partitioner/edge_based_graph_reader.hpp b/include/partitioner/edge_based_graph_reader.hpp index 06b8b6ae9..99f111435 100644 --- a/include/partitioner/edge_based_graph_reader.hpp +++ b/include/partitioner/edge_based_graph_reader.hpp @@ -173,8 +173,8 @@ graphToEdges(const DynamicEdgeBasedGraph &edge_based_graph) for (auto edge : edge_based_graph.GetAdjacentEdgeRange(node)) { const auto &data = edge_based_graph.GetEdgeData(edge); - // we only need to save the forward edges, since the read method will - // convert from forward to bi-directional edges again + // we only need to save the forward edges, since the read method + // will convert from forward to bi-directional edges again if (data.forward) { auto target = edge_based_graph.GetTarget(edge); @@ -191,7 +191,7 @@ graphToEdges(const DynamicEdgeBasedGraph &edge_based_graph) return edges; } -inline DynamicEdgeBasedGraph LoadEdgeBasedGraph(const boost::filesystem::path &path) +inline DynamicEdgeBasedGraph LoadEdgeBasedGraph(const std::filesystem::path &path) { EdgeID number_of_edge_based_nodes; std::vector edges; diff --git a/include/partitioner/files.hpp b/include/partitioner/files.hpp index 9bd74447b..ff8c47499 100644 --- a/include/partitioner/files.hpp +++ b/include/partitioner/files.hpp @@ -10,7 +10,7 @@ namespace osrm::partitioner::files // read .osrm.partition file template -inline void readPartition(const boost::filesystem::path &path, MultiLevelPartitionT &mlp) +inline void readPartition(const std::filesystem::path &path, MultiLevelPartitionT &mlp) { static_assert(std::is_same::value || std::is_same::value, @@ -24,7 +24,7 @@ inline void readPartition(const boost::filesystem::path &path, MultiLevelPartiti // writes .osrm.partition file template -inline void writePartition(const boost::filesystem::path &path, const MultiLevelPartitionT &mlp) +inline void writePartition(const std::filesystem::path &path, const MultiLevelPartitionT &mlp) { static_assert(std::is_same::value || std::is_same::value, @@ -38,7 +38,7 @@ inline void writePartition(const boost::filesystem::path &path, const MultiLevel // reads .osrm.cells file template -inline void readCells(const boost::filesystem::path &path, CellStorageT &storage) +inline void readCells(const std::filesystem::path &path, CellStorageT &storage) { static_assert(std::is_same::value || std::is_same::value, @@ -52,7 +52,7 @@ inline void readCells(const boost::filesystem::path &path, CellStorageT &storage // writes .osrm.cells file template -inline void writeCells(const boost::filesystem::path &path, CellStorageT &storage) +inline void writeCells(const std::filesystem::path &path, CellStorageT &storage) { static_assert(std::is_same::value || std::is_same::value, @@ -66,7 +66,7 @@ inline void writeCells(const boost::filesystem::path &path, CellStorageT &storag // reads .osrm.mldgr file template -inline void readGraph(const boost::filesystem::path &path, +inline void readGraph(const std::filesystem::path &path, MultiLevelGraphT &graph, std::uint32_t &connectivity_checksum) { @@ -80,7 +80,7 @@ inline void readGraph(const boost::filesystem::path &path, // writes .osrm.mldgr file template -inline void writeGraph(const boost::filesystem::path &path, +inline void writeGraph(const std::filesystem::path &path, const MultiLevelGraphT &graph, const std::uint32_t connectivity_checksum) { diff --git a/include/partitioner/partitioner_config.hpp b/include/partitioner/partitioner_config.hpp index 021ed1b4e..3bffdd515 100644 --- a/include/partitioner/partitioner_config.hpp +++ b/include/partitioner/partitioner_config.hpp @@ -1,9 +1,8 @@ #ifndef OSRM_PARTITIONER_CONFIG_HPP #define OSRM_PARTITIONER_CONFIG_HPP -#include - #include +#include #include #include "storage/io_config.hpp" @@ -29,7 +28,7 @@ struct PartitionerConfig final : storage::IOConfig { } - void UseDefaultOutputNames(const boost::filesystem::path &base) + void UseDefaultOutputNames(const std::filesystem::path &base) { IOConfig::UseDefaultOutputNames(base); } diff --git a/include/storage/io.hpp b/include/storage/io.hpp index 21095576e..cebb180cd 100644 --- a/include/storage/io.hpp +++ b/include/storage/io.hpp @@ -10,14 +10,13 @@ #include "util/log.hpp" #include "util/version.hpp" -#include -#include #include #include #include - #include #include +#include +#include #include #include #include @@ -35,11 +34,11 @@ class FileReader }; FileReader(const std::string &filename, const FingerprintFlag flag) - : FileReader(boost::filesystem::path(filename), flag) + : FileReader(std::filesystem::path(filename), flag) { } - FileReader(const boost::filesystem::path &filepath_, const FingerprintFlag flag) + FileReader(const std::filesystem::path &filepath_, const FingerprintFlag flag) : filepath(filepath_), fingerprint(flag) { input_stream.open(filepath, std::ios::binary); @@ -58,14 +57,14 @@ class FileReader std::size_t GetSize() { - const boost::filesystem::path path(filepath); + const std::filesystem::path path(filepath); try { - return std::size_t(boost::filesystem::file_size(path)) - + return std::size_t(std::filesystem::file_size(path)) - ((fingerprint == FingerprintFlag::VerifyFingerprint) ? sizeof(util::FingerPrint) : 0); } - catch (const boost::filesystem::filesystem_error &ex) + catch (const std::filesystem::filesystem_error &ex) { std::cout << ex.what() << std::endl; throw; @@ -196,8 +195,8 @@ class FileReader } private: - const boost::filesystem::path filepath; - boost::filesystem::ifstream input_stream; + const std::filesystem::path filepath; + std::ifstream input_stream; FingerprintFlag fingerprint; }; @@ -211,11 +210,11 @@ class FileWriter }; FileWriter(const std::string &filename, const FingerprintFlag flag) - : FileWriter(boost::filesystem::path(filename), flag) + : FileWriter(std::filesystem::path(filename), flag) { } - FileWriter(const boost::filesystem::path &filepath_, const FingerprintFlag flag) + FileWriter(const std::filesystem::path &filepath_, const FingerprintFlag flag) : filepath(filepath_), fingerprint(flag) { output_stream.open(filepath, std::ios::binary); @@ -284,8 +283,8 @@ class FileWriter } private: - const boost::filesystem::path filepath; - boost::filesystem::ofstream output_stream; + const std::filesystem::path filepath; + std::ofstream output_stream; FingerprintFlag fingerprint; }; diff --git a/include/storage/io_config.hpp b/include/storage/io_config.hpp index ee6478638..c3db9214a 100644 --- a/include/storage/io_config.hpp +++ b/include/storage/io_config.hpp @@ -3,19 +3,19 @@ #include "util/exception.hpp" -#include #include -#include -#include + +#include +#include #include namespace osrm::storage { struct IOConfig { - IOConfig(std::vector required_input_files_, - std::vector optional_input_files_, - std::vector output_files_) + IOConfig(std::vector required_input_files_, + std::vector optional_input_files_, + std::vector output_files_) : required_input_files(std::move(required_input_files_)), optional_input_files(std::move(optional_input_files_)), output_files(std::move(output_files_)) @@ -24,7 +24,7 @@ struct IOConfig bool IsValid() const; std::vector GetMissingFiles() const; - boost::filesystem::path GetPath(const std::string &fileName) const + std::filesystem::path GetPath(const std::string &fileName) const { if (!IsConfigured(fileName, required_input_files) && !IsConfigured(fileName, optional_input_files) && !IsConfigured(fileName, output_files)) @@ -40,11 +40,11 @@ struct IOConfig return IsConfigured(fileName, required_input_files); } - boost::filesystem::path base_path; + std::filesystem::path base_path; protected: // Infer the base path from the path of the .osrm file - void UseDefaultOutputNames(const boost::filesystem::path &base) + void UseDefaultOutputNames(const std::filesystem::path &base) { // potentially strip off the .osrm (or other) extensions for // determining the base path= @@ -67,7 +67,7 @@ struct IOConfig private: static bool IsConfigured(const std::string &fileName, - const std::vector &paths) + const std::vector &paths) { for (auto &path : paths) { @@ -80,9 +80,9 @@ struct IOConfig return false; } - std::vector required_input_files; - std::vector optional_input_files; - std::vector output_files; + std::vector required_input_files; + std::vector optional_input_files; + std::vector output_files; }; } // namespace osrm::storage diff --git a/include/storage/shared_memory.hpp b/include/storage/shared_memory.hpp index 3300728b9..1b33d4ebe 100644 --- a/include/storage/shared_memory.hpp +++ b/include/storage/shared_memory.hpp @@ -5,8 +5,6 @@ #include "util/exception_utils.hpp" #include "util/log.hpp" -#include -#include #include #ifndef _WIN32 #include @@ -23,6 +21,8 @@ #include #include +#include +#include #include #include "storage/shared_memory_ownership.hpp" @@ -32,10 +32,10 @@ namespace osrm::storage struct OSRMLockFile { - template boost::filesystem::path operator()(const IdentifierT &id) + template std::filesystem::path operator()(const IdentifierT &id) { - boost::filesystem::path temp_dir = boost::filesystem::temp_directory_path(); - boost::filesystem::path lock_file = temp_dir / ("osrm-" + std::to_string(id) + ".lock"); + std::filesystem::path temp_dir = std::filesystem::temp_directory_path(); + std::filesystem::path lock_file = temp_dir / ("osrm-" + std::to_string(id) + ".lock"); return lock_file; } }; @@ -51,7 +51,7 @@ class SharedMemory SharedMemory &operator=(const SharedMemory &) = delete; template - SharedMemory(const boost::filesystem::path &lock_file, + SharedMemory(const std::filesystem::path &lock_file, const IdentifierT id, const uint64_t size = 0) : key(lock_file.string().c_str(), id) @@ -202,7 +202,7 @@ class SharedMemory void *Ptr() const { return region.get_address(); } std::size_t Size() const { return region.get_size(); } - SharedMemory(const boost::filesystem::path &lock_file, const int id, const uint64_t size = 0) + SharedMemory(const std::filesystem::path &lock_file, const int id, const uint64_t size = 0) { sprintf(key, "%s.%d", "osrm.lock", id); if (0 == size) @@ -290,7 +290,7 @@ std::unique_ptr makeSharedMemory(const IdentifierT &id, const uint try { LockFileT lock_file; - if (!boost::filesystem::exists(lock_file(id))) + if (!std::filesystem::exists(lock_file(id))) { if (0 == size) { @@ -298,7 +298,7 @@ std::unique_ptr makeSharedMemory(const IdentifierT &id, const uint } else { - boost::filesystem::ofstream ofs(lock_file(id)); + std::ofstream ofs(lock_file(id)); } } return std::make_unique(lock_file(id), id, size); diff --git a/include/storage/storage.hpp b/include/storage/storage.hpp index b9c22e509..839d2ae8a 100644 --- a/include/storage/storage.hpp +++ b/include/storage/storage.hpp @@ -32,15 +32,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "storage/shared_datatype.hpp" #include "storage/storage_config.hpp" -#include - +#include #include #include namespace osrm::storage { -void populateLayoutFromFile(const boost::filesystem::path &path, storage::BaseDataLayout &layout); +void populateLayoutFromFile(const std::filesystem::path &path, storage::BaseDataLayout &layout); class Storage { @@ -51,10 +50,10 @@ class Storage void PopulateStaticData(const SharedDataIndex &index); void PopulateUpdatableData(const SharedDataIndex &index); void PopulateLayout(storage::BaseDataLayout &layout, - const std::vector> &files); + const std::vector> &files); std::string PopulateLayoutWithRTree(storage::BaseDataLayout &layout); - std::vector> GetUpdatableFiles(); - std::vector> GetStaticFiles(); + std::vector> GetUpdatableFiles(); + std::vector> GetStaticFiles(); private: StorageConfig config; diff --git a/include/storage/storage_config.hpp b/include/storage/storage_config.hpp index 3059cddd0..3cc1720d4 100644 --- a/include/storage/storage_config.hpp +++ b/include/storage/storage_config.hpp @@ -28,22 +28,23 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef STORAGE_CONFIG_HPP #define STORAGE_CONFIG_HPP -#include - #include "storage/io_config.hpp" #include "osrm/datasets.hpp" +#include +#include #include +#include namespace osrm::storage { std::istream &operator>>(std::istream &in, FeatureDataset &datasets); -static std::vector +static std::vector GetRequiredFiles(const std::vector &disabled_feature_dataset) { - std::set required{ + std::set required{ ".osrm.datasource_names", ".osrm.ebg_nodes", ".osrm.edges", @@ -82,7 +83,7 @@ GetRequiredFiles(const std::vector &disabled_feature_da } } - return std::vector(required.begin(), required.end()); + return std::vector(required.begin(), required.end()); ; } @@ -94,7 +95,7 @@ GetRequiredFiles(const std::vector &disabled_feature_da struct StorageConfig final : IOConfig { - StorageConfig(const boost::filesystem::path &base, + StorageConfig(const std::filesystem::path &base, const std::vector &disabled_feature_datasets_ = {}) : StorageConfig(disabled_feature_datasets_) { diff --git a/include/storage/tar.hpp b/include/storage/tar.hpp index 9da7bcca0..a942adf25 100644 --- a/include/storage/tar.hpp +++ b/include/storage/tar.hpp @@ -7,7 +7,7 @@ #include "util/integer_range.hpp" #include "util/version.hpp" -#include +#include extern "C" { @@ -19,7 +19,7 @@ namespace osrm::storage::tar namespace detail { inline void -checkMTarError(int error_code, const boost::filesystem::path &filepath, const std::string &name) +checkMTarError(int error_code, const std::filesystem::path &filepath, const std::string &name) { switch (error_code) { @@ -78,7 +78,7 @@ class FileReader HasNoFingerprint }; - FileReader(const boost::filesystem::path &path, FingerprintFlag flag) : path(path) + FileReader(const std::filesystem::path &path, FingerprintFlag flag) : path(path) { auto ret = mtar_open(&handle, path.string().c_str(), "r"); detail::checkMTarError(ret, path, ""); @@ -204,7 +204,7 @@ class FileReader return true; } - boost::filesystem::path path; + std::filesystem::path path; mtar_t handle; }; @@ -217,7 +217,7 @@ class FileWriter HasNoFingerprint }; - FileWriter(const boost::filesystem::path &path, FingerprintFlag flag) : path(path) + FileWriter(const std::filesystem::path &path, FingerprintFlag flag) : path(path) { auto ret = mtar_open(&handle, path.string().c_str(), "w"); detail::checkMTarError(ret, path, ""); @@ -305,7 +305,7 @@ class FileWriter WriteFrom("osrm_fingerprint.meta", fingerprint); } - boost::filesystem::path path; + std::filesystem::path path; mtar_t handle; }; } // namespace osrm::storage::tar diff --git a/include/storage/view_factory.hpp b/include/storage/view_factory.hpp index 88e99e8e8..00223f57c 100644 --- a/include/storage/view_factory.hpp +++ b/include/storage/view_factory.hpp @@ -202,7 +202,7 @@ inline auto make_search_tree_view(const SharedDataIndex &index, const std::strin const char *path = index.template GetBlockPtr(name + "/file_index_path"); - if (!boost::filesystem::exists(boost::filesystem::path{path})) + if (!std::filesystem::exists(std::filesystem::path{path})) { throw util::exception("Could not load " + std::string(path) + "Does the leaf file exist?" + SOURCE_REF); diff --git a/include/updater/csv_file_parser.hpp b/include/updater/csv_file_parser.hpp index c8d5d4cde..bac8d9bfc 100644 --- a/include/updater/csv_file_parser.hpp +++ b/include/updater/csv_file_parser.hpp @@ -12,12 +12,12 @@ #include #include -#include #include #include #include #include +#include #include #include @@ -100,7 +100,7 @@ template struct CSVFilesParser std::vector> result; try { - if (boost::filesystem::file_size(filename) == 0) + if (std::filesystem::file_size(filename) == 0) return result; boost::iostreams::mapped_file_source mmap(filename); diff --git a/include/updater/updater_config.hpp b/include/updater/updater_config.hpp index cc4d2ad4b..d1dab9f3c 100644 --- a/include/updater/updater_config.hpp +++ b/include/updater/updater_config.hpp @@ -28,14 +28,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef OSRM_UPDATER_UPDATER_CONFIG_HPP #define OSRM_UPDATER_UPDATER_CONFIG_HPP -#include - -#include -#include - #include "storage/io_config.hpp" #include "storage/storage_config.hpp" +#include +#include +#include + namespace osrm::updater { @@ -59,7 +58,7 @@ struct UpdaterConfig final : storage::IOConfig { } - void UseDefaultOutputNames(const boost::filesystem::path &base) + void UseDefaultOutputNames(const std::filesystem::path &base) { IOConfig::UseDefaultOutputNames(base); } diff --git a/include/util/lua_util.hpp b/include/util/lua_util.hpp index 75811db1d..92c2961c8 100644 --- a/include/util/lua_util.hpp +++ b/include/util/lua_util.hpp @@ -8,7 +8,7 @@ extern "C" #include } -#include +#include #include @@ -20,7 +20,7 @@ namespace osrm::util // See http://lua-users.org/wiki/PackagePath for details on the package.path syntax. inline void luaAddScriptFolderToLoadPath(lua_State *lua_state, const char *file_name) { - boost::filesystem::path profile_path = boost::filesystem::canonical(file_name); + std::filesystem::path profile_path = std::filesystem::canonical(file_name); std::string folder = profile_path.parent_path().generic_string(); const std::string lua_code = "package.path = \"" + folder + "/?.lua;\" .. package.path"; luaL_dostring(lua_state, lua_code.c_str()); diff --git a/include/util/mmap_file.hpp b/include/util/mmap_file.hpp index cdee22763..e06a53c90 100644 --- a/include/util/mmap_file.hpp +++ b/include/util/mmap_file.hpp @@ -5,20 +5,22 @@ #include "util/exception_utils.hpp" #include "util/vector_view.hpp" -#include #include +#include + namespace osrm::util { namespace detail { template -util::vector_view mmapFile(const boost::filesystem::path &file, MmapContainerT &mmap_container) +util::vector_view mmapFile(const std::filesystem::path &file, MmapContainerT &mmap_container) { try { - mmap_container.open(file); + auto path_string = file.string(); + mmap_container.open(path_string); std::size_t num_objects = mmap_container.size() / sizeof(T); auto data_ptr = mmap_container.data(); BOOST_ASSERT(reinterpret_cast(data_ptr) % alignof(T) == 0); @@ -33,9 +35,8 @@ util::vector_view mmapFile(const boost::filesystem::path &file, MmapContainer } template -util::vector_view mmapFile(const boost::filesystem::path &file, - MmapContainerT &mmap_container, - const std::size_t size) +util::vector_view +mmapFile(const std::filesystem::path &file, MmapContainerT &mmap_container, const std::size_t size) { try { @@ -61,21 +62,21 @@ util::vector_view mmapFile(const boost::filesystem::path &file, } // namespace detail template -util::vector_view mmapFile(const boost::filesystem::path &file, +util::vector_view mmapFile(const std::filesystem::path &file, boost::iostreams::mapped_file_source &mmap_container) { return detail::mmapFile(file, mmap_container); } template -util::vector_view mmapFile(const boost::filesystem::path &file, +util::vector_view mmapFile(const std::filesystem::path &file, boost::iostreams::mapped_file &mmap_container) { return detail::mmapFile(file, mmap_container); } template -util::vector_view mmapFile(const boost::filesystem::path &file, +util::vector_view mmapFile(const std::filesystem::path &file, boost::iostreams::mapped_file &mmap_container, std::size_t size) { diff --git a/include/util/mmap_tar.hpp b/include/util/mmap_tar.hpp index 2fa7679ea..a70d5fba8 100644 --- a/include/util/mmap_tar.hpp +++ b/include/util/mmap_tar.hpp @@ -15,7 +15,7 @@ namespace osrm::util using DataRange = std::pair; using DataMap = std::unordered_map; -inline DataMap mmapTarFile(const boost::filesystem::path &path, +inline DataMap mmapTarFile(const std::filesystem::path &path, boost::iostreams::mapped_file_source ®ion) { DataMap map; diff --git a/include/util/static_rtree.hpp b/include/util/static_rtree.hpp index dc88da233..f9d72c144 100644 --- a/include/util/static_rtree.hpp +++ b/include/util/static_rtree.hpp @@ -20,7 +20,6 @@ #include "storage/shared_memory_ownership.hpp" #include -#include #include #include @@ -30,6 +29,7 @@ #include #include +#include #include #include #include @@ -271,7 +271,7 @@ class StaticRTree // Construct a packed Hilbert-R-Tree with Kamel-Faloutsos algorithm [1] explicit StaticRTree(const std::vector &input_data_vector, const Vector &coordinate_list, - const boost::filesystem::path &on_disk_file_name) + const std::filesystem::path &on_disk_file_name) : m_coordinate_list(coordinate_list.data(), coordinate_list.size()) { const auto element_count = input_data_vector.size(); @@ -458,7 +458,7 @@ class StaticRTree * Constructs an empty RTree for de-serialization. */ template > - explicit StaticRTree(const boost::filesystem::path &on_disk_file_name, + explicit StaticRTree(const std::filesystem::path &on_disk_file_name, const Vector &coordinate_list) : m_coordinate_list(coordinate_list.data(), coordinate_list.size()), m_objects(mmapFile(on_disk_file_name, m_objects_region)) @@ -473,7 +473,7 @@ class StaticRTree */ explicit StaticRTree(Vector search_tree_, Vector tree_level_starts, - const boost::filesystem::path &on_disk_file_name, + const std::filesystem::path &on_disk_file_name, const Vector &coordinate_list) : m_search_tree(std::move(search_tree_)), m_coordinate_list(coordinate_list.data(), coordinate_list.size()), diff --git a/include/util/timezones.hpp b/include/util/timezones.hpp index 8105aaae4..e144662de 100644 --- a/include/util/timezones.hpp +++ b/include/util/timezones.hpp @@ -3,13 +3,13 @@ #include "util/log.hpp" -#include #include #include #include #include +#include #include namespace osrm::updater @@ -32,7 +32,7 @@ class Timezoner Timezoner() = default; Timezoner(const char geojson[], std::time_t utc_time_now); - Timezoner(const boost::filesystem::path &tz_shapes_filename, std::time_t utc_time_now); + Timezoner(const std::filesystem::path &tz_shapes_filename, std::time_t utc_time_now); std::optional operator()(const point_t &point) const; diff --git a/scripts/ci/windows-build.bat b/scripts/ci/windows-build.bat new file mode 100644 index 000000000..a74bdd2c2 --- /dev/null +++ b/scripts/ci/windows-build.bat @@ -0,0 +1,96 @@ +@ECHO OFF +SETLOCAL +SET EL=0 + +ECHO NUMBER_OF_PROCESSORS^: %NUMBER_OF_PROCESSORS% + +SET PROJECT_DIR=%CD% +SET CONFIGURATION=Release + +mkdir build +IF %ERRORLEVEL% NEQ 0 GOTO ERROR +cd build +IF %ERRORLEVEL% NEQ 0 GOTO ERROR +cmake -DENABLE_CONAN=ON -DENABLE_NODE_BINDINGS=ON -DCMAKE_BUILD_TYPE=%CONFIGURATION% -G "Visual Studio 17 2022" .. +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +msbuild OSRM.sln ^ +/p:Configuration=%CONFIGURATION% ^ +/p:Platform=x64 ^ +/p:nowarn="4244;4267;4365;4456;4514;4625;4626;4710;4711;4820;5026;5027" ^ +/t:rebuild ^ +/p:BuildInParallel=true ^ +/m:%NUMBER_OF_PROCESSORS% ^ +/toolsversion:Current ^ +/clp:Verbosity=quiet ^ +/nologo +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +CD %PROJECT_DIR%\build +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +ECHO running extractor-tests.exe ... +unit_tests\%CONFIGURATION%\extractor-tests.exe +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +ECHO running contractor-tests.exe ... +unit_tests\%CONFIGURATION%\contractor-tests.exe +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +ECHO running engine-tests.exe ... +unit_tests\%CONFIGURATION%\engine-tests.exe +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +ECHO running util-tests.exe ... +unit_tests\%CONFIGURATION%\util-tests.exe +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +ECHO running server-tests.exe ... +unit_tests\%CONFIGURATION%\server-tests.exe +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +ECHO running partitioner-tests.exe ... +unit_tests\%CONFIGURATION%\partitioner-tests.exe +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +ECHO running customizer-tests.exe ... +unit_tests\%CONFIGURATION%\customizer-tests.exe +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +SET test_region=monaco +SET test_region_ch=ch\monaco +SET test_region_corech=corech\monaco +SET test_region_mld=mld\monaco +SET test_osm=%test_region%.osm.pbf +COPY %PROJECT_DIR%\test\data\%test_region%.osm.pbf %test_osm% +%CONFIGURATION%\osrm-extract.exe -p %PROJECT_DIR%\profiles\car.lua %test_osm% +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +MKDIR ch +XCOPY %test_region%.osrm.* ch\ +XCOPY %test_region%.osrm ch\ +MKDIR corech +XCOPY %test_region%.osrm.* corech\ +XCOPY %test_region%.osrm corech\ +MKDIR mld +XCOPY %test_region%.osrm.* mld\ +XCOPY %test_region%.osrm mld\ +%CONFIGURATION%\osrm-contract.exe %test_region_ch%.osrm +%CONFIGURATION%\osrm-contract.exe --core 0.8 %test_region_corech%.osrm +%CONFIGURATION%\osrm-partition.exe %test_region_mld%.osrm +%CONFIGURATION%\osrm-customize.exe %test_region_mld%.osrm +XCOPY /Y ch\*.* ..\test\data\ch\ +XCOPY /Y corech\*.* ..\test\data\corech\ +XCOPY /Y mld\*.* ..\test\data\mld\ +unit_tests\%CONFIGURATION%\library-tests.exe +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +:ERROR +ECHO ~~~~~~~~~~~~~~~~~~~~~~ ERROR %~f0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +ECHO ERRORLEVEL^: %ERRORLEVEL% +SET EL=%ERRORLEVEL% + +:DONE +ECHO ~~~~~~~~~~~~~~~~~~~~~~ DONE %~f0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +EXIT /b %EL% diff --git a/src/benchmarks/static_rtree.cpp b/src/benchmarks/static_rtree.cpp index eaa784ea8..8346b2e9b 100644 --- a/src/benchmarks/static_rtree.cpp +++ b/src/benchmarks/static_rtree.cpp @@ -11,11 +11,10 @@ #include "util/serialization.hpp" #include "util/timing_util.hpp" +#include #include #include -#include - namespace osrm::benchmarks { diff --git a/src/engine/datafacade/mmap_memory_allocator.cpp b/src/engine/datafacade/mmap_memory_allocator.cpp index e16331fa0..75482637b 100644 --- a/src/engine/datafacade/mmap_memory_allocator.cpp +++ b/src/engine/datafacade/mmap_memory_allocator.cpp @@ -44,7 +44,7 @@ MMapMemoryAllocator::MMapMemoryAllocator(const storage::StorageConfig &config) for (const auto &file : files) { - if (boost::filesystem::exists(file.second)) + if (std::filesystem::exists(file.second)) { std::unique_ptr layout = std::make_unique(); diff --git a/src/extractor/compressed_edge_container.cpp b/src/extractor/compressed_edge_container.cpp index 916b1d489..ddfafcef4 100644 --- a/src/extractor/compressed_edge_container.cpp +++ b/src/extractor/compressed_edge_container.cpp @@ -2,10 +2,11 @@ #include "util/log.hpp" #include -#include -#include #include +#include +#include +#include #include #include diff --git a/src/extractor/location_dependent_data.cpp b/src/extractor/location_dependent_data.cpp index 475f15f20..f72e921b8 100644 --- a/src/extractor/location_dependent_data.cpp +++ b/src/extractor/location_dependent_data.cpp @@ -7,17 +7,17 @@ #include #include -#include #include #include +#include #include #include namespace osrm::extractor { -LocationDependentData::LocationDependentData(const std::vector &file_paths) +LocationDependentData::LocationDependentData(const std::vector &file_paths) { std::vector bounding_boxes; for (const auto &path : file_paths) @@ -32,12 +32,12 @@ LocationDependentData::LocationDependentData(const std::vector &bounding_boxes) + const std::filesystem::path &file_path, std::vector &bounding_boxes) { if (file_path.empty()) return; - if (!boost::filesystem::exists(file_path) || !boost::filesystem::is_regular_file(file_path)) + if (!std::filesystem::exists(file_path) || !std::filesystem::is_regular_file(file_path)) { throw osrm::util::exception(std::string("File with location-dependent data ") + file_path.string() + " does not exists"); diff --git a/src/extractor/raster_source.cpp b/src/extractor/raster_source.cpp index cda4f78f5..bc889dcae 100644 --- a/src/extractor/raster_source.cpp +++ b/src/extractor/raster_source.cpp @@ -103,8 +103,8 @@ int RasterContainer::LoadRasterSource(const std::string &path_string, util::Log() << "[source loader] Loading from " << path_string << " ... "; TIMER_START(loading_source); - boost::filesystem::path filepath(path_string); - if (!boost::filesystem::exists(filepath)) + std::filesystem::path filepath(path_string); + if (!std::filesystem::exists(filepath)) { throw util::RuntimeError( path_string, ErrorCode::FileOpenError, SOURCE_REF, "File not found"); diff --git a/src/extractor/scripting_environment_lua.cpp b/src/extractor/scripting_environment_lua.cpp index 200efa8b1..8232f0476 100644 --- a/src/extractor/scripting_environment_lua.cpp +++ b/src/extractor/scripting_environment_lua.cpp @@ -113,7 +113,7 @@ void handle_lua_error(const sol::protected_function_result &luares) Sol2ScriptingEnvironment::Sol2ScriptingEnvironment( const std::string &file_name, - const std::vector &location_dependent_data_paths) + const std::vector &location_dependent_data_paths) : file_name(file_name), location_dependent_data(location_dependent_data_paths) { util::Log() << "Using script " << file_name; diff --git a/src/partitioner/partitioner.cpp b/src/partitioner/partitioner.cpp index dbed86bb1..abc9757eb 100644 --- a/src/partitioner/partitioner.cpp +++ b/src/partitioner/partitioner.cpp @@ -19,16 +19,15 @@ #include "util/json_container.hpp" #include "util/log.hpp" #include "util/mmap_file.hpp" - -#include -#include -#include +#include "util/timing_util.hpp" #include -#include #include -#include "util/timing_util.hpp" +#include +#include +#include +#include namespace osrm::partitioner { @@ -171,11 +170,11 @@ int Partitioner::Run(const PartitionerConfig &config) extractor::files::writeManeuverOverrides(filename, maneuver_overrides, node_sequences); } - if (boost::filesystem::exists(config.GetPath(".osrm.hsgr"))) + if (std::filesystem::exists(config.GetPath(".osrm.hsgr"))) { util::Log(logWARNING) << "Found existing .osrm.hsgr file, removing. You need to re-run " "osrm-contract after osrm-partition."; - boost::filesystem::remove(config.GetPath(".osrm.hsgr")); + std::filesystem::remove(config.GetPath(".osrm.hsgr")); } TIMER_STOP(renumber); util::Log() << "Renumbered data in " << TIMER_SEC(renumber) << " seconds"; diff --git a/src/storage/io_config.cpp b/src/storage/io_config.cpp index 4ad810295..8e21cded6 100644 --- a/src/storage/io_config.cpp +++ b/src/storage/io_config.cpp @@ -2,14 +2,12 @@ #include "util/log.hpp" -#include -#include -#include +#include namespace osrm::storage { -namespace fs = boost::filesystem; +namespace fs = std::filesystem; bool IOConfig::IsValid() const { diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index d3e2041e5..a8e8ba33b 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -23,13 +23,11 @@ #endif #include -#include -#include #include #include #include - +#include #include #include #include @@ -164,7 +162,7 @@ bool swapData(Monitor &monitor, } } // namespace -void populateLayoutFromFile(const boost::filesystem::path &path, storage::BaseDataLayout &layout) +void populateLayoutFromFile(const std::filesystem::path &path, storage::BaseDataLayout &layout) { tar::FileReader reader(path, tar::FileReader::VerifyFingerprint); @@ -190,11 +188,11 @@ int Storage::Run(int max_wait, const std::string &dataset_name, bool only_metric util::LogPolicy::GetInstance().Unmute(); - boost::filesystem::path lock_path = - boost::filesystem::temp_directory_path() / "osrm-datastore.lock"; - if (!boost::filesystem::exists(lock_path)) + std::filesystem::path lock_path = + std::filesystem::temp_directory_path() / "osrm-datastore.lock"; + if (!std::filesystem::exists(lock_path)) { - boost::filesystem::ofstream ofs(lock_path); + std::ofstream ofs(lock_path); } boost::interprocess::file_lock file_lock(lock_path.string().c_str()); @@ -257,7 +255,7 @@ int Storage::Run(int max_wait, const std::string &dataset_name, bool only_metric std::unique_ptr static_layout = std::make_unique(); Storage::PopulateLayoutWithRTree(*static_layout); - std::vector> files = Storage::GetStaticFiles(); + std::vector> files = Storage::GetStaticFiles(); Storage::PopulateLayout(*static_layout, files); auto static_handle = setupRegion(shared_register, *static_layout); regions.push_back({static_handle.data_ptr, std::move(static_layout)}); @@ -266,7 +264,7 @@ int Storage::Run(int max_wait, const std::string &dataset_name, bool only_metric std::unique_ptr updatable_layout = std::make_unique(); - std::vector> files = Storage::GetUpdatableFiles(); + std::vector> files = Storage::GetUpdatableFiles(); Storage::PopulateLayout(*updatable_layout, files); auto updatable_handle = setupRegion(shared_register, *updatable_layout); regions.push_back({updatable_handle.data_ptr, std::move(updatable_layout)}); @@ -285,12 +283,12 @@ int Storage::Run(int max_wait, const std::string &dataset_name, bool only_metric return EXIT_SUCCESS; } -std::vector> Storage::GetStaticFiles() +std::vector> Storage::GetStaticFiles() { constexpr bool IS_REQUIRED = true; constexpr bool IS_OPTIONAL = false; - std::vector> files = { + std::vector> files = { {IS_OPTIONAL, config.GetPath(".osrm.cells")}, {IS_OPTIONAL, config.GetPath(".osrm.partition")}, {IS_REQUIRED, config.GetPath(".osrm.ebg_nodes")}, @@ -310,7 +308,7 @@ std::vector> Storage::GetStaticFiles() for (const auto &file : files) { - if (file.first == IS_REQUIRED && !boost::filesystem::exists(file.second)) + if (file.first == IS_REQUIRED && !std::filesystem::exists(file.second)) { throw util::exception("Could not find required file(s): " + std::get<1>(file).string()); } @@ -319,12 +317,12 @@ std::vector> Storage::GetStaticFiles() return files; } -std::vector> Storage::GetUpdatableFiles() +std::vector> Storage::GetUpdatableFiles() { constexpr bool IS_REQUIRED = true; constexpr bool IS_OPTIONAL = false; - std::vector> files = { + std::vector> files = { {IS_OPTIONAL, config.GetPath(".osrm.mldgr")}, {IS_OPTIONAL, config.GetPath(".osrm.cell_metrics")}, {IS_OPTIONAL, config.GetPath(".osrm.hsgr")}, @@ -335,7 +333,7 @@ std::vector> Storage::GetUpdatableFiles for (const auto &file : files) { - if (file.first == IS_REQUIRED && !boost::filesystem::exists(file.second)) + if (file.first == IS_REQUIRED && !std::filesystem::exists(file.second)) { throw util::exception("Could not find required file(s): " + std::get<1>(file).string()); } @@ -347,9 +345,9 @@ std::vector> Storage::GetUpdatableFiles std::string Storage::PopulateLayoutWithRTree(storage::BaseDataLayout &layout) { // Figure out the path to the rtree file (it's not a tar file) - auto absolute_file_index_path = boost::filesystem::absolute(config.GetPath(".osrm.fileIndex")); + auto absolute_file_index_path = std::filesystem::absolute(config.GetPath(".osrm.fileIndex")); - // Convert the boost::filesystem::path object into a plain string + // Convert the std::filesystem::path object into a plain string // that can then be stored as a member of an allocator object auto rtree_filename = absolute_file_index_path.string(); @@ -366,11 +364,11 @@ std::string Storage::PopulateLayoutWithRTree(storage::BaseDataLayout &layout) * in that big block. It updates the fields in the layout parameter. */ void Storage::PopulateLayout(storage::BaseDataLayout &layout, - const std::vector> &files) + const std::vector> &files) { for (const auto &file : files) { - if (boost::filesystem::exists(file.second)) + if (std::filesystem::exists(file.second)) { populateLayoutFromFile(file.second, layout); } @@ -389,7 +387,7 @@ void Storage::PopulateStaticData(const SharedDataIndex &index) file_index_path_ptr + index.GetBlockSize("/common/rtree/file_index_path"), 0); const auto absolute_file_index_path = - boost::filesystem::absolute(config.GetPath(".osrm.fileIndex")).string(); + std::filesystem::absolute(config.GetPath(".osrm.fileIndex")).string(); BOOST_ASSERT(static_cast(index.GetBlockSize( "/common/rtree/file_index_path")) >= absolute_file_index_path.size()); std::copy( @@ -482,13 +480,13 @@ void Storage::PopulateStaticData(const SharedDataIndex &index) metric_name = profile_properties_ptr->GetWeightName(); } - if (boost::filesystem::exists(config.GetPath(".osrm.partition"))) + if (std::filesystem::exists(config.GetPath(".osrm.partition"))) { auto mlp = make_partition_view(index, "/mld/multilevelpartition"); partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp); } - if (boost::filesystem::exists(config.GetPath(".osrm.cells"))) + if (std::filesystem::exists(config.GetPath(".osrm.cells"))) { auto storage = make_cell_storage_view(index, "/mld/cellstorage"); partitioner::files::readCells(config.GetPath(".osrm.cells"), storage); @@ -541,7 +539,7 @@ void Storage::PopulateUpdatableData(const SharedDataIndex &index) metric_name = properties.GetWeightName(); } - if (boost::filesystem::exists(config.GetPath(".osrm.hsgr"))) + if (std::filesystem::exists(config.GetPath(".osrm.hsgr"))) { const std::string metric_prefix = "/ch/metrics/" + metric_name; auto contracted_metric = make_contracted_metric_view(index, metric_prefix); @@ -567,7 +565,7 @@ void Storage::PopulateUpdatableData(const SharedDataIndex &index) } } - if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics"))) + if (std::filesystem::exists(config.GetPath(".osrm.cell_metrics"))) { auto exclude_metrics = make_cell_metric_view(index, "/mld/metrics/" + metric_name); std::unordered_map> metrics = { @@ -576,7 +574,7 @@ void Storage::PopulateUpdatableData(const SharedDataIndex &index) customizer::files::readCellMetrics(config.GetPath(".osrm.cell_metrics"), metrics); } - if (boost::filesystem::exists(config.GetPath(".osrm.mldgr"))) + if (std::filesystem::exists(config.GetPath(".osrm.mldgr"))) { auto graph_view = make_multi_level_graph_view(index, "/mld/multilevelgraph"); std::uint32_t graph_connectivity_checksum = 0; diff --git a/src/tools/components.cpp b/src/tools/components.cpp index edd9a3242..1f55a3afa 100644 --- a/src/tools/components.cpp +++ b/src/tools/components.cpp @@ -10,7 +10,6 @@ #include "util/tarjan_scc.hpp" #include "util/typedefs.hpp" -#include #include #include @@ -19,6 +18,7 @@ #include #include +#include #include #include #include @@ -118,7 +118,7 @@ int main(int argc, char *argv[]) const std::string inpath{argv[1]}; const std::string outpath{argv[2]}; - if (boost::filesystem::exists(outpath)) + if (std::filesystem::exists(outpath)) { util::Log(logWARNING) << "Components file " << outpath << " already exists"; return EXIT_FAILURE; diff --git a/src/tools/contract.cpp b/src/tools/contract.cpp index 289ec671b..6f7900e62 100644 --- a/src/tools/contract.cpp +++ b/src/tools/contract.cpp @@ -6,12 +6,12 @@ #include "util/timezones.hpp" #include "util/version.hpp" -#include #include #include #include #include +#include #include #include #include @@ -86,7 +86,7 @@ return_code parseArguments(int argc, boost::program_options::options_description hidden_options("Hidden options"); hidden_options.add_options()( "input,i", - boost::program_options::value(&contractor_config.base_path), + boost::program_options::value(&contractor_config.base_path), "Input file in .osm, .osm.bz2 or .osm.pbf format"); // positional option @@ -99,7 +99,7 @@ return_code parseArguments(int argc, const auto *executable = argv[0]; boost::program_options::options_description visible_options( - "Usage: " + boost::filesystem::path(executable).filename().string() + + "Usage: " + std::filesystem::path(executable).filename().string() + " [options]"); visible_options.add(generic_options).add(config_options); diff --git a/src/tools/customize.cpp b/src/tools/customize.cpp index f77b88795..0f3f8cb09 100644 --- a/src/tools/customize.cpp +++ b/src/tools/customize.cpp @@ -5,9 +5,9 @@ #include "util/meminfo.hpp" #include "util/version.hpp" -#include #include +#include #include #include @@ -76,7 +76,7 @@ return_code parseArguments(int argc, boost::program_options::options_description hidden_options("Hidden options"); hidden_options.add_options()( "input,i", - boost::program_options::value(&customization_config.base_path), + boost::program_options::value(&customization_config.base_path), "Input base file path"); // positional option @@ -89,7 +89,7 @@ return_code parseArguments(int argc, const auto *executable = argv[0]; boost::program_options::options_description visible_options( - boost::filesystem::path(executable).filename().string() + " [options]"); + std::filesystem::path(executable).filename().string() + " [options]"); visible_options.add(generic_options).add(config_options); // parse command line options diff --git a/src/tools/extract.cpp b/src/tools/extract.cpp index 77964727b..c39ecdc2a 100644 --- a/src/tools/extract.cpp +++ b/src/tools/extract.cpp @@ -2,19 +2,17 @@ #include "osrm/extractor.hpp" #include "osrm/extractor_config.hpp" #include "util/log.hpp" +#include "util/meminfo.hpp" #include "util/version.hpp" -#include #include - #include #include +#include #include #include #include -#include "util/meminfo.hpp" - using namespace osrm; enum class return_code : unsigned @@ -40,7 +38,7 @@ return_code parseArguments(int argc, boost::program_options::options_description config_options("Configuration"); config_options.add_options()( "profile,p", - boost::program_options::value(&extractor_config.profile_path) + boost::program_options::value(&extractor_config.profile_path) ->default_value("profiles/car.lua"), "Path to LUA routing profile")( "data_version,d", @@ -67,7 +65,7 @@ return_code parseArguments(int argc, ->default_value(false), "Save conditional restrictions found during extraction to disk for use " "during contraction")("location-dependent-data", - boost::program_options::value>( + boost::program_options::value>( &extractor_config.location_dependent_data_paths) ->composing(), "GeoJSON files with location-dependent data")( @@ -88,7 +86,7 @@ return_code parseArguments(int argc, boost::program_options::options_description hidden_options("Hidden options"); hidden_options.add_options()( "input,i", - boost::program_options::value(&extractor_config.input_path), + boost::program_options::value(&extractor_config.input_path), "Input file in .osm, .osm.bz2 or .osm.pbf format")( "generate-edge-lookup", boost::program_options::bool_switch(&dummy)->implicit_value(true)->default_value(false), @@ -104,7 +102,7 @@ return_code parseArguments(int argc, const auto *executable = argv[0]; boost::program_options::options_description visible_options( - boost::filesystem::path(executable).filename().string() + + std::filesystem::path(executable).filename().string() + " [options]"); visible_options.add(generic_options).add(config_options); @@ -176,14 +174,14 @@ try return EXIT_FAILURE; } - if (!boost::filesystem::is_regular_file(extractor_config.input_path)) + if (!std::filesystem::is_regular_file(extractor_config.input_path)) { util::Log(logERROR) << "Input file " << extractor_config.input_path.string() << " not found!"; return EXIT_FAILURE; } - if (!boost::filesystem::is_regular_file(extractor_config.profile_path)) + if (!std::filesystem::is_regular_file(extractor_config.profile_path)) { util::Log(logERROR) << "Profile " << extractor_config.profile_path.string() << " not found!"; diff --git a/src/tools/io-benchmark.cpp b/src/tools/io-benchmark.cpp index 80db12926..966ab9879 100644 --- a/src/tools/io-benchmark.cpp +++ b/src/tools/io-benchmark.cpp @@ -3,9 +3,6 @@ #include "util/log.hpp" #include "util/timing_util.hpp" -#include -#include - #include #include #include @@ -15,6 +12,8 @@ #include #include +#include +#include #include #include #include @@ -45,7 +44,7 @@ void runStatistics(std::vector &timings_vector, Statistics &stats) } } // namespace osrm::tools -boost::filesystem::path test_path; +std::filesystem::path test_path; int main(int argc, char *argv[]) { @@ -66,7 +65,7 @@ int main(int argc, char *argv[]) return -1; } - test_path = boost::filesystem::path(argv[1]); + test_path = std::filesystem::path(argv[1]); test_path /= "osrm.tst"; osrm::util::Log(logDEBUG) << "temporary file: " << test_path.string(); @@ -74,7 +73,7 @@ int main(int argc, char *argv[]) if (2 == argc) { // create file to test - if (boost::filesystem::exists(test_path)) + if (std::filesystem::exists(test_path)) { throw osrm::util::exception("Data file already exists: " + test_path.string() + SOURCE_REF); @@ -120,7 +119,7 @@ int main(int argc, char *argv[]) else { // Run Non-Cached I/O benchmarks - if (!boost::filesystem::exists(test_path)) + if (!std::filesystem::exists(test_path)) { throw osrm::util::exception("data file does not exist" + SOURCE_REF); } @@ -298,9 +297,9 @@ int main(int argc, char *argv[]) << "max: " << stats.max << "ms, " << "dev: " << stats.dev << "ms"; - if (boost::filesystem::exists(test_path)) + if (std::filesystem::exists(test_path)) { - boost::filesystem::remove(test_path); + std::filesystem::remove(test_path); osrm::util::Log(logDEBUG) << "removing temporary files"; } } diff --git a/src/tools/partition.cpp b/src/tools/partition.cpp index acc06a045..fb983d9ba 100644 --- a/src/tools/partition.cpp +++ b/src/tools/partition.cpp @@ -8,10 +8,10 @@ #include "util/version.hpp" #include -#include #include #include +#include #include #include #include @@ -119,7 +119,7 @@ return_code parseArguments(int argc, boost::program_options::options_description hidden_options("Hidden options"); hidden_options.add_options()( "input,i", - boost::program_options::value(&config.base_path), + boost::program_options::value(&config.base_path), "Input base file path"); // positional option @@ -132,7 +132,7 @@ return_code parseArguments(int argc, const auto *executable = argv[0]; boost::program_options::options_description visible_options( - boost::filesystem::path(executable).filename().string() + " [options]"); + std::filesystem::path(executable).filename().string() + " [options]"); visible_options.add(generic_options).add(config_options); // parse command line options @@ -216,9 +216,9 @@ try return EXIT_FAILURE; } - auto check_file = [](const boost::filesystem::path &path) + auto check_file = [](const std::filesystem::path &path) { - if (!boost::filesystem::is_regular_file(path)) + if (!std::filesystem::is_regular_file(path)) { util::Log(logERROR) << "Input file " << path << " not found!"; return false; diff --git a/src/tools/routed.cpp b/src/tools/routed.cpp index f4c229cba..07dcf864c 100644 --- a/src/tools/routed.cpp +++ b/src/tools/routed.cpp @@ -12,7 +12,6 @@ #include #include -#include #include #include @@ -22,6 +21,7 @@ #include #include +#include #include #include #include @@ -102,7 +102,7 @@ void validate(boost::any &v, const std::vector &values, double *, d // generate boost::program_options object for the routing part inline unsigned generateServerProgramOptions(const int argc, const char *argv[], - boost::filesystem::path &base_path, + std::filesystem::path &base_path, std::string &ip_address, int &ip_port, bool &trial, @@ -110,8 +110,8 @@ inline unsigned generateServerProgramOptions(const int argc, int &requested_thread_num, short &keepalive_timeout) { - using boost::filesystem::path; using boost::program_options::value; + using std::filesystem::path; const auto hardware_threads = std::max(1, std::thread::hardware_concurrency()); @@ -148,7 +148,7 @@ inline unsigned generateServerProgramOptions(const int argc, value(&config.use_shared_memory)->implicit_value(true)->default_value(false), "Load data from shared memory") // ("memory_file", - value(&config.memory_file), + value(&config.memory_file), "DEPRECATED: Will behave the same as --mmap.")( "mmap,m", value(&config.use_mmap)->implicit_value(true)->default_value(false), @@ -192,7 +192,7 @@ inline unsigned generateServerProgramOptions(const int argc, // hidden options, will be allowed on command line, but will not be shown to the user boost::program_options::options_description hidden_options("Hidden options"); hidden_options.add_options()( - "base,b", value(&base_path), "base path to .osrm file"); + "base,b", value(&base_path), "base path to .osrm file"); // positional option boost::program_options::positional_options_description positional_options; @@ -204,7 +204,7 @@ inline unsigned generateServerProgramOptions(const int argc, const auto *executable = argv[0]; boost::program_options::options_description visible_options( - boost::filesystem::path(executable).filename().string() + " []"); + std::filesystem::path(executable).filename().string() + " []"); visible_options.add(generic_options).add(config_options); // parse command line options @@ -267,7 +267,7 @@ try int ip_port; EngineConfig config; - boost::filesystem::path base_path; + std::filesystem::path base_path; int requested_thread_num = 1; short keepalive_timeout = 5; diff --git a/src/tools/store.cpp b/src/tools/store.cpp index e821610e8..6cd555f04 100644 --- a/src/tools/store.cpp +++ b/src/tools/store.cpp @@ -11,11 +11,11 @@ #include "util/version.hpp" #include -#include #include #include #include +#include using namespace osrm; @@ -97,7 +97,7 @@ void springClean() bool generateDataStoreOptions(const int argc, const char *argv[], std::string &verbosity, - boost::filesystem::path &base_path, + std::filesystem::path &base_path, int &max_wait, std::string &dataset_name, bool &list_datasets, @@ -155,7 +155,7 @@ bool generateDataStoreOptions(const int argc, // hidden options, will be allowed on command line but will not be shown to the user boost::program_options::options_description hidden_options("Hidden options"); hidden_options.add_options()("base,b", - boost::program_options::value(&base_path), + boost::program_options::value(&base_path), "base path to .osrm file"); // positional option @@ -168,7 +168,7 @@ bool generateDataStoreOptions(const int argc, const auto *executable = argv[0]; boost::program_options::options_description visible_options( - boost::filesystem::path(executable).filename().string() + " [] "); + std::filesystem::path(executable).filename().string() + " [] "); visible_options.add(generic_options).add(config_options); // print help options if no infile is specified @@ -242,7 +242,7 @@ try util::LogPolicy::GetInstance().Unmute(); std::string verbosity; - boost::filesystem::path base_path; + std::filesystem::path base_path; int max_wait = -1; std::string dataset_name; bool list_datasets = false; diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index e82f854f8..165d2387d 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -25,7 +25,6 @@ #include "util/typedefs.hpp" #include -#include #include #include @@ -38,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -408,7 +408,7 @@ void saveDatasourcesNames(const UpdaterConfig &config) // for rendering in the debug tiles. for (auto const &name : config.segment_speed_lookup_paths) { - sources.SetSourceName(source, boost::filesystem::path(name).stem().string()); + sources.SetSourceName(source, std::filesystem::path(name).stem().string()); source++; } diff --git a/src/util/timezones.cpp b/src/util/timezones.cpp index 3fd527d69..3352cfaf0 100644 --- a/src/util/timezones.cpp +++ b/src/util/timezones.cpp @@ -3,13 +3,11 @@ #include "util/geojson_validation.hpp" #include "util/log.hpp" -#include -#include - #include #include #include +#include #include #include #include @@ -40,7 +38,7 @@ Timezoner::Timezoner(const char geojson[], std::time_t utc_time_now) LoadLocalTimesRTree(doc, utc_time_now); } -Timezoner::Timezoner(const boost::filesystem::path &tz_shapes_filename, std::time_t utc_time_now) +Timezoner::Timezoner(const std::filesystem::path &tz_shapes_filename, std::time_t utc_time_now) { util::Log() << "Time zone validation based on UTC time : " << utc_time_now; diff --git a/unit_tests/common/temporary_file.hpp b/unit_tests/common/temporary_file.hpp index 68f2538ca..244550161 100644 --- a/unit_tests/common/temporary_file.hpp +++ b/unit_tests/common/temporary_file.hpp @@ -1,16 +1,39 @@ #ifndef UNIT_TESTS_TEMPORARY_FILE_HPP #define UNIT_TESTS_TEMPORARY_FILE_HPP -#include +#include +#include +#include +#include + +inline std::string random_string(std::string::size_type length) +{ + static auto &chrs = "0123456789" + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + thread_local static std::mt19937 rg{std::random_device{}()}; + thread_local static std::uniform_int_distribution pick( + 0, sizeof(chrs) - 2); + + std::string s; + s.reserve(length); + + while (length--) + { + s += chrs[pick(rg)]; + } + return s; +} struct TemporaryFile { - TemporaryFile() : path(boost::filesystem::unique_path()) {} + TemporaryFile() : path(std::filesystem::temp_directory_path() / random_string(8)) {} TemporaryFile(const std::string &path) : path(path) {} - ~TemporaryFile() { boost::filesystem::remove(path); } + ~TemporaryFile() { std::filesystem::remove(path); } - boost::filesystem::path path; + std::filesystem::path path; }; #endif // UNIT_TESTS_TEMPORARY_FILE_HPP diff --git a/unit_tests/extractor/location_dependent_data_tests.cpp b/unit_tests/extractor/location_dependent_data_tests.cpp index 86f36eb72..d70a48bbd 100644 --- a/unit_tests/extractor/location_dependent_data_tests.cpp +++ b/unit_tests/extractor/location_dependent_data_tests.cpp @@ -1,11 +1,14 @@ #include "extractor/location_dependent_data.hpp" #include "../common/range_tools.hpp" +#include "../common/temporary_file.hpp" -#include #include +#include +#include #include +#include #include BOOST_AUTO_TEST_SUITE(location_dependent_data_tests) @@ -16,14 +19,13 @@ using point_t = LocationDependentData::point_t; struct LocationDataFixture { - LocationDataFixture(const std::string &json) : temporary_file(boost::filesystem::unique_path()) + LocationDataFixture(const std::string &json) { - std::ofstream file(temporary_file.string()); + std::ofstream file(temporary_file.path.string()); file << json; } - ~LocationDataFixture() { remove(temporary_file); } - boost::filesystem::path temporary_file; + TemporaryFile temporary_file; }; BOOST_AUTO_TEST_CASE(polygon_tests) @@ -50,7 +52,7 @@ BOOST_AUTO_TEST_CASE(polygon_tests) } ]})json"); - LocationDependentData data({fixture.temporary_file}); + LocationDependentData data({fixture.temporary_file.path}); BOOST_CHECK(data.GetPropertyIndexes(point_t(0, 0)).empty()); CHECK_EQUAL_RANGE(data.GetPropertyIndexes(point_t(1, 1)), 0); @@ -86,7 +88,7 @@ BOOST_AUTO_TEST_CASE(multy_polygon_tests) } ]})json"); - LocationDependentData data({fixture.temporary_file}); + LocationDependentData data({fixture.temporary_file.path}); BOOST_CHECK(data.GetPropertyIndexes(point_t(0, 2)).empty()); BOOST_CHECK(data.GetPropertyIndexes(point_t(0, -3)).empty()); @@ -117,7 +119,7 @@ BOOST_AUTO_TEST_CASE(polygon_merging_tests) } ]})json"); - LocationDependentData data({fixture.temporary_file}); + LocationDependentData data({fixture.temporary_file.path}); CHECK_EQUAL_RANGE(data.GetPropertyIndexes(point_t(-3, 3)), 0); CHECK_EQUAL_RANGE(data.GetPropertyIndexes(point_t(-3, 1)), 0); @@ -153,7 +155,7 @@ BOOST_AUTO_TEST_CASE(staircase_polygon) } ]})json"); - LocationDependentData data({fixture.temporary_file}); + LocationDependentData data({fixture.temporary_file.path}); // all corners BOOST_CHECK(!data.GetPropertyIndexes(point_t(0, 0)).empty()); diff --git a/unit_tests/extractor/name_table.cpp b/unit_tests/extractor/name_table.cpp index 7f06e9298..c4ebd0ac5 100644 --- a/unit_tests/extractor/name_table.cpp +++ b/unit_tests/extractor/name_table.cpp @@ -3,9 +3,9 @@ #include "../common/temporary_file.hpp" -#include #include +#include #include #include #include diff --git a/unit_tests/extractor/raster_source.cpp b/unit_tests/extractor/raster_source.cpp index 1721f3d39..8ecbca688 100644 --- a/unit_tests/extractor/raster_source.cpp +++ b/unit_tests/extractor/raster_source.cpp @@ -4,9 +4,10 @@ #include -#include #include +#include + BOOST_AUTO_TEST_SUITE(raster_source) using namespace osrm; diff --git a/unit_tests/storage/data_layout.cpp b/unit_tests/storage/data_layout.cpp index b9ef8e8ba..de0d77656 100644 --- a/unit_tests/storage/data_layout.cpp +++ b/unit_tests/storage/data_layout.cpp @@ -3,10 +3,10 @@ #include "../common/range_tools.hpp" #include "../common/temporary_file.hpp" -#include #include #include +#include BOOST_AUTO_TEST_SUITE(data_layout) diff --git a/unit_tests/storage/serialization.cpp b/unit_tests/storage/serialization.cpp index 4b38e43d1..d369a9841 100644 --- a/unit_tests/storage/serialization.cpp +++ b/unit_tests/storage/serialization.cpp @@ -5,9 +5,9 @@ #include "../common/range_tools.hpp" #include "../common/temporary_file.hpp" -#include #include +#include #include BOOST_AUTO_TEST_SUITE(serialization) diff --git a/unit_tests/updater/timezoner.cpp b/unit_tests/updater/timezoner.cpp index 9f4a3744a..473fa4be0 100644 --- a/unit_tests/updater/timezoner.cpp +++ b/unit_tests/updater/timezoner.cpp @@ -2,9 +2,10 @@ #include "util/geojson_validation.hpp" #include "util/timezones.hpp" -#include #include +#include + BOOST_AUTO_TEST_SUITE(timezoner) using namespace osrm; @@ -22,7 +23,7 @@ BOOST_AUTO_TEST_CASE(timezoner_test) std::time_t now = time(0); BOOST_CHECK_NO_THROW(Timezoner tz(json, now)); - boost::filesystem::path test_path(TEST_DATA_DIR "/test.geojson"); + std::filesystem::path test_path(TEST_DATA_DIR "/test.geojson"); BOOST_CHECK_NO_THROW(Timezoner tz(test_path, now)); // missing opening bracket diff --git a/unit_tests/util/serialization.cpp b/unit_tests/util/serialization.cpp index dffb0ea03..be2b963a7 100644 --- a/unit_tests/util/serialization.cpp +++ b/unit_tests/util/serialization.cpp @@ -3,9 +3,10 @@ #include "../common/range_tools.hpp" #include "../common/temporary_file.hpp" -#include #include +#include + BOOST_AUTO_TEST_SUITE(serialization) using namespace osrm; diff --git a/unit_tests/util/static_rtree.cpp b/unit_tests/util/static_rtree.cpp index c1b9b373a..d2e29fa2e 100644 --- a/unit_tests/util/static_rtree.cpp +++ b/unit_tests/util/static_rtree.cpp @@ -227,7 +227,7 @@ void sampling_verify_rtree(RTreeT &rtree, } template -auto make_rtree(const boost::filesystem::path &path, FixtureT &fixture) +auto make_rtree(const std::filesystem::path &path, FixtureT &fixture) { return RTreeT(fixture.edges, fixture.coords, path); } From 3285ce3b2f1fcd7e8b71403a3e20ba98fd8dc149 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Fri, 21 Jun 2024 08:51:56 +0200 Subject: [PATCH 12/17] Configure stale bot (#6963) --- .github/workflows/stale.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 000000000..72cf595ee --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,24 @@ +name: 'Close stale issues' +on: + schedule: + - cron: '30 1 * * *' # every day at 1:30am +permissions: + issues: write + pull-requests: write + +jobs: + stale: + runs-on: ubuntu-24.04 + steps: + - uses: actions/stale@v9 + with: + stale-issue-message: 'This issue seems to be stale. It will be closed in 30 days if no further activity occurs.' + stale-pr-message: 'This PR seems to be stale. Is it still relevant?' + days-before-issue-stale: 180 # 6 months + days-before-issue-close: 30 # 1 month + days-before-pr-stale: 180 # 6 months + days-before-pr-close: -1 # never close PRs + exempt-issue-labels: 'Do Not Stale,Feature Request,Performance,Bug Report,CI,Starter Task,Refactor,Guidance' + + + From d9ce9cf780777c96d74e0430b7c6f1dee6930f9a Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Fri, 21 Jun 2024 19:04:34 +0200 Subject: [PATCH 13/17] Get rid of unused Boost dependencies (#6960) --- CHANGELOG.md | 1 + CMakeLists.txt | 16 +++------------- docker/Dockerfile-alpine | 4 +--- docker/Dockerfile-debian | 5 ++--- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5b4c9662..a0abc407c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - NodeJS: - CHANGED: Use node-api instead of NAN. [#6452](https://github.com/Project-OSRM/osrm-backend/pull/6452) - Misc: + - CHANGED: Get rid of unused Boost dependencies. [#6960](https://github.com/Project-OSRM/osrm-backend/pull/6960) - CHANGED: Apply micro-optimisation for Table & Trip APIs. [#6949](https://github.com/Project-OSRM/osrm-backend/pull/6949) - CHANGED: Apply micro-optimisation for Route API. [#6948](https://github.com/Project-OSRM/osrm-backend/pull/6948) - CHANGED: Apply micro-optimisation for Match API. [#6945](https://github.com/Project-OSRM/osrm-backend/pull/6945) diff --git a/CMakeLists.txt b/CMakeLists.txt index eac368946..532782067 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,7 +122,7 @@ include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/include/) include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/include/) include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/sol2/include) -set(BOOST_COMPONENTS date_time chrono filesystem iostreams program_options regex system thread unit_test_framework) +set(BOOST_COMPONENTS date_time iostreams program_options thread unit_test_framework) configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/include/util/version.hpp.in @@ -348,8 +348,7 @@ if(ENABLE_CONAN) GENERATORS cmake_find_package json # json generator generates a conanbuildinfo.json in the build folder so (non-CMake) projects can easily parse OSRM's dependencies KEEP_RPATHS NO_OUTPUT_DIRS - OPTIONS boost:filesystem_version=3 # https://stackoverflow.com/questions/73392648/error-with-boost-filesystem-version-in-cmake - boost:without_stacktrace=True # Apple Silicon cross-compilation fails without it + OPTIONS boost:without_stacktrace=True # Apple Silicon cross-compilation fails without it BUILD missing ) @@ -378,14 +377,10 @@ if(ENABLE_CONAN) set(Boost_USE_STATIC_LIBS ON) find_package(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS}) set(Boost_DATE_TIME_LIBRARY "${Boost_date_time_LIB_TARGETS}") - set(Boost_CHRONO_LIBRARY "${Boost_chrono_LIB_TARGETS}") set(Boost_PROGRAM_OPTIONS_LIBRARY "${Boost_program_options_LIB_TARGETS}") - set(Boost_FILESYSTEM_LIBRARY "${Boost_filesystem_LIB_TARGETS}") set(Boost_IOSTREAMS_LIBRARY "${Boost_iostreams_LIB_TARGETS}") set(Boost_THREAD_LIBRARY "${Boost_thread_LIB_TARGETS}") - set(Boost_SYSTEM_LIBRARY "${Boost_system_LIB_TARGETS}") set(Boost_ZLIB_LIBRARY "${Boost_zlib_LIB_TARGETS}") - set(Boost_REGEX_LIBRARY "${Boost_regex_LIB_TARGETS}") set(Boost_UNIT_TEST_FRAMEWORK_LIBRARY "${Boost_unit_test_framework_LIB_TARGETS}") @@ -458,7 +453,6 @@ add_dependency_includes(${ZLIB_INCLUDE_DIRS}) add_dependency_defines(-DBOOST_SPIRIT_USE_PHOENIX_V3) add_dependency_defines(-DBOOST_RESULT_OF_USE_DECLTYPE) -add_dependency_defines(-DBOOST_FILESYSTEM_NO_DEPRECATED) # Workaround for https://github.com/boostorg/phoenix/issues/111 add_dependency_defines(-DBOOST_PHOENIX_STL_TUPLE_H_) @@ -468,11 +462,8 @@ include_directories(SYSTEM ${DEPENDENCIES_INCLUDE_DIRS}) set(BOOST_BASE_LIBRARIES ${Boost_DATE_TIME_LIBRARY} - ${Boost_CHRONO_LIBRARY} - ${Boost_FILESYSTEM_LIBRARY} ${Boost_IOSTREAMS_LIBRARY} - ${Boost_THREAD_LIBRARY} - ${Boost_SYSTEM_LIBRARY}) + ${Boost_THREAD_LIBRARY}) set(BOOST_ENGINE_LIBRARIES ${Boost_ZLIB_LIBRARY} @@ -491,7 +482,6 @@ endif() set(EXTRACTOR_LIBRARIES ${BZIP2_LIBRARIES} - ${Boost_REGEX_LIBRARY} ${BOOST_BASE_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${EXPAT_LIBRARIES} diff --git a/docker/Dockerfile-alpine b/docker/Dockerfile-alpine index c636b1619..42c20ae38 100644 --- a/docker/Dockerfile-alpine +++ b/docker/Dockerfile-alpine @@ -47,9 +47,7 @@ COPY --from=builder /usr/local /usr/local COPY --from=builder /opt /opt RUN apk add --no-cache \ - boost-program_options boost-regex \ - boost-date_time boost-chrono boost-filesystem \ - boost-iostreams boost-system boost-thread \ + boost-program_options boost-date_time boost-iostreams boost-thread \ expat lua5.4 onetbb && \ ldconfig /usr/local/lib diff --git a/docker/Dockerfile-debian b/docker/Dockerfile-debian index ca4ae165d..cfec9cbed 100644 --- a/docker/Dockerfile-debian +++ b/docker/Dockerfile-debian @@ -49,9 +49,8 @@ COPY --from=builder /usr/local /usr/local COPY --from=builder /opt /opt RUN apt-get update && \ - apt-get install -y --no-install-recommends libboost-program-options1.81.0 libboost-regex1.81.0 \ - libboost-date-time1.81.0 libboost-chrono1.81.0 libboost-filesystem1.81.0 \ - libboost-iostreams1.81.0 libboost-system1.81.0 libboost-thread1.81.0 \ + apt-get install -y --no-install-recommends \ + libboost-program-options1.81.0 libboost-date-time1.81.0 libboost-iostreams1.81.0 libboost-thread1.81.0 \ expat liblua5.4-0 && \ rm -rf /var/lib/apt/lists/* && \ # add /usr/local/lib to ldconfig to allow loading libraries from there From e8da3d92312e0deaf1ca853848718477bed132e8 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Fri, 21 Jun 2024 21:43:27 +0200 Subject: [PATCH 14/17] Calculate confidence interval for benchmark measurements (#6950) --- .github/workflows/osrm-backend.yml | 27 +- scripts/ci/e2e_benchmark.py | 72 ++- scripts/ci/run_benchmarks.sh | 115 +++-- src/benchmarks/bench.cpp | 712 ++++++++++++++++------------- src/benchmarks/match.cpp | 2 +- src/benchmarks/route.cpp | 34 -- 6 files changed, 553 insertions(+), 409 deletions(-) diff --git a/.github/workflows/osrm-backend.yml b/.github/workflows/osrm-backend.yml index ee8af27b9..b743ee942 100644 --- a/.github/workflows/osrm-backend.yml +++ b/.github/workflows/osrm-backend.yml @@ -700,15 +700,6 @@ jobs: mkdir -p $HOME/.ccache ccache --zero-stats ccache --max-size=256M - - name: Build PR Branch - run: | - mkdir -p pr/build - cd pr/build - cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release .. - make -j$(nproc) - make -j$(nproc) benchmarks - cd .. - make -C test/data - name: Checkout Base Branch uses: actions/checkout@v4 with: @@ -723,9 +714,23 @@ jobs: make -j$(nproc) benchmarks cd .. make -C test/data - - name: Run Benchmarks + - name: Build PR Branch run: | - ./pr/scripts/ci/run_benchmarks.sh base pr + mkdir -p pr/build + cd pr/build + cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release .. + make -j$(nproc) + make -j$(nproc) benchmarks + cd .. + make -C test/data + - name: Run PR Benchmarks + run: | + ./pr/scripts/ci/run_benchmarks.sh -f $(pwd)/pr -r $(pwd)/pr_results -s $(pwd)/pr -b $(pwd)/pr/build -o ~/data.osm.pbf -g ~/gps_traces.csv + - name: Run Base Benchmarks + run: | + # we intentionally use scripts from PR branch to be able to update them and see results in the same PR + ./pr/scripts/ci/run_benchmarks.sh -f $(pwd)/base -r $(pwd)/base_results -s $(pwd)/pr -b $(pwd)/base/build -o ~/data.osm.pbf -g ~/gps_traces.csv + - name: Post Benchmark Results run: | python3 pr/scripts/ci/post_benchmark_results.py base_results pr_results diff --git a/scripts/ci/e2e_benchmark.py b/scripts/ci/e2e_benchmark.py index 4fc900eb9..02a0b59b8 100644 --- a/scripts/ci/e2e_benchmark.py +++ b/scripts/ci/e2e_benchmark.py @@ -1,5 +1,4 @@ import requests -import sys import random from collections import defaultdict import os @@ -8,12 +7,13 @@ import numpy as np import time import argparse + class BenchmarkRunner: - def __init__(self): + def __init__(self, gps_traces_file_path): self.coordinates = [] self.tracks = defaultdict(list) - gps_traces_file_path = os.path.expanduser('~/gps_traces.csv') + gps_traces_file_path = os.path.expanduser(gps_traces_file_path) with open(gps_traces_file_path, 'r') as file: reader = csv.DictReader(file) for row in reader: @@ -36,10 +36,9 @@ class BenchmarkRunner: response = requests.get(url) end_time = time.time() if response.status_code != 200: - if benchmark_name == 'match': - code = response.json()['code'] - if code == 'NoSegment' or code == 'NoMatch': - continue + code = response.json()['code'] + if code in ['NoSegment', 'NoMatch', 'NoRoute', 'NoTrips']: + continue raise Exception(f"Error: {response.status_code} {response.text}") times.append((end_time - start_time) * 1000) # convert to ms @@ -54,7 +53,7 @@ class BenchmarkRunner: end_coord = f"{end[1]:.6f},{end[0]:.6f}" return f"{host}/route/v1/driving/{start_coord};{end_coord}?overview=full&steps=true" elif benchmark_name == 'table': - num_coords = random.randint(3, 100) + num_coords = random.randint(3, 12) selected_coords = random.sample(self.coordinates, num_coords) coords_str = ";".join([f"{coord[1]:.6f},{coord[0]:.6f}" for coord in selected_coords]) return f"{host}/table/v1/driving/{coords_str}" @@ -77,26 +76,63 @@ class BenchmarkRunner: else: raise Exception(f"Unknown benchmark: {benchmark_name}") +def bootstrap_confidence_interval(data, num_samples=1000, confidence_level=0.95): + means = [] + for _ in range(num_samples): + sample = np.random.choice(data, size=len(data), replace=True) + means.append(np.mean(sample)) + lower_bound = np.percentile(means, (1 - confidence_level) / 2 * 100) + upper_bound = np.percentile(means, (1 + confidence_level) / 2 * 100) + mean = np.mean(means) + return mean, lower_bound, upper_bound + +def calculate_confidence_interval(data): + mean, lower, upper = bootstrap_confidence_interval(data) + min_value = np.min(data) + return mean, (upper - lower) / 2, min_value + + def main(): parser = argparse.ArgumentParser(description='Run GPS benchmark tests.') parser.add_argument('--host', type=str, required=True, help='Host URL') parser.add_argument('--method', type=str, required=True, choices=['route', 'table', 'match', 'nearest', 'trip'], help='Benchmark method') parser.add_argument('--num_requests', type=int, required=True, help='Number of requests to perform') + parser.add_argument('--iterations', type=int, required=True, help='Number of iterations to run the benchmark') + parser.add_argument('--gps_traces_file_path', type=str, required=True, help='Path to the GPS traces file') args = parser.parse_args() - random.seed(42) + np.random.seed(42) - runner = BenchmarkRunner() - times = runner.run(args.method, args.host, args.num_requests) + runner = BenchmarkRunner(args.gps_traces_file_path) + + all_times = [] + for _ in range(args.iterations): + random.seed(42) + times = runner.run(args.method, args.host, args.num_requests) + all_times.append(times) + all_times = np.asarray(all_times) - print(f'Total: {np.sum(times)}ms') - print(f"Min time: {np.min(times)}ms") - print(f"Mean time: {np.mean(times)}ms") - print(f"Median time: {np.median(times)}ms") - print(f"95th percentile: {np.percentile(times, 95)}ms") - print(f"99th percentile: {np.percentile(times, 99)}ms") - print(f"Max time: {np.max(times)}ms") + assert all_times.shape == (args.iterations, all_times.shape[1]) + + + total_time, total_ci, total_best = calculate_confidence_interval(np.sum(all_times, axis=1)) + ops_per_sec, ops_per_sec_ci, ops_per_sec_best = calculate_confidence_interval(float(all_times.shape[1]) / np.sum(all_times / 1000, axis=1)) + min_time, min_ci, _ = calculate_confidence_interval(np.min(all_times, axis=1)) + mean_time, mean_ci, _ = calculate_confidence_interval(np.mean(all_times, axis=1)) + median_time, median_ci, _ = calculate_confidence_interval(np.median(all_times, axis=1)) + perc_95_time, perc_95_ci, _ = calculate_confidence_interval(np.percentile(all_times, 95, axis=1)) + perc_99_time, perc_99_ci, _ = calculate_confidence_interval(np.percentile(all_times, 99, axis=1)) + max_time, max_ci, _ = calculate_confidence_interval(np.max(all_times, axis=1)) + + print(f'Ops: {ops_per_sec:.2f} ± {ops_per_sec_ci:.2f} ops/s. Best: {ops_per_sec_best:.2f} ops/s') + print(f'Total: {total_time:.2f}ms ± {total_ci:.2f}ms. Best: {total_best:.2f}ms') + print(f"Min time: {min_time:.2f}ms ± {min_ci:.2f}ms") + print(f"Mean time: {mean_time:.2f}ms ± {mean_ci:.2f}ms") + print(f"Median time: {median_time:.2f}ms ± {median_ci:.2f}ms") + print(f"95th percentile: {perc_95_time:.2f}ms ± {perc_95_ci:.2f}ms") + print(f"99th percentile: {perc_99_time:.2f}ms ± {perc_99_ci:.2f}ms") + print(f"Max time: {max_time:.2f}ms ± {max_ci:.2f}ms") if __name__ == '__main__': main() diff --git a/scripts/ci/run_benchmarks.sh b/scripts/ci/run_benchmarks.sh index 2affff0f7..a0f32ece5 100755 --- a/scripts/ci/run_benchmarks.sh +++ b/scripts/ci/run_benchmarks.sh @@ -1,72 +1,121 @@ #!/bin/bash set -eou pipefail +function usage { + echo "Usage: $0 -f -r -s -b -o -g " + exit 1 +} + +while getopts ":f:r:s:b:o:g:" opt; do + case $opt in + f) FOLDER="$OPTARG" + ;; + r) RESULTS_FOLDER="$OPTARG" + ;; + s) SCRIPTS_FOLDER="$OPTARG" + ;; + b) BINARIES_FOLDER="$OPTARG" + ;; + o) OSM_PBF="$OPTARG" + ;; + g) GPS_TRACES="$OPTARG" + ;; + \?) echo "Invalid option -$OPTARG" >&2 + usage + ;; + :) echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done + +if [ -z "${FOLDER:-}" ] || [ -z "${RESULTS_FOLDER:-}" ] || [ -z "${SCRIPTS_FOLDER:-}" ] || [ -z "${BINARIES_FOLDER:-}" ] || [ -z "${OSM_PBF:-}" ] || [ -z "${GPS_TRACES:-}" ]; then + usage +fi + function measure_peak_ram_and_time { COMMAND=$1 OUTPUT_FILE=$2 + if [ "$(uname)" == "Darwin" ]; then + # on macOS time has different parameters, so simply run command on macOS + $COMMAND > /dev/null 2>&1 + else + OUTPUT=$(/usr/bin/time -f "%e %M" $COMMAND 2>&1 | tail -n 1) - OUTPUT=$(/usr/bin/time -f "%e %M" $COMMAND 2>&1 | tail -n 1) - - TIME=$(echo $OUTPUT | awk '{print $1}') - PEAK_RAM_KB=$(echo $OUTPUT | awk '{print $2}') - PEAK_RAM_MB=$(echo "scale=2; $PEAK_RAM_KB / 1024" | bc) - echo "Time: ${TIME}s Peak RAM: ${PEAK_RAM_MB}MB" > $OUTPUT_FILE + TIME=$(echo $OUTPUT | awk '{print $1}') + PEAK_RAM_KB=$(echo $OUTPUT | awk '{print $2}') + PEAK_RAM_MB=$(echo "scale=2; $PEAK_RAM_KB / 1024" | bc) + echo "Time: ${TIME}s Peak RAM: ${PEAK_RAM_MB}MB" > $OUTPUT_FILE + fi } function run_benchmarks_for_folder { - echo "Running benchmarks for $1" - - FOLDER=$1 - RESULTS_FOLDER=$2 - SCRIPTS_FOLDER=$3 - mkdir -p $RESULTS_FOLDER - BENCHMARKS_FOLDER="$FOLDER/build/src/benchmarks" + BENCHMARKS_FOLDER="$BINARIES_FOLDER/src/benchmarks" + echo "Running match-bench MLD" + $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench" + echo "Running match-bench CH" + $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench" + echo "Running route-bench MLD" + $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench" + echo "Running route-bench CH" + $BENCHMARKS_FOLDER/route-bench "$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench" + echo "Running alias" + $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench" + echo "Running json-render-bench" + $BENCHMARKS_FOLDER/json-render-bench "$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench" + echo "Running packedvector-bench" + $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench" + echo "Running rtree-bench" + $BENCHMARKS_FOLDER/rtree-bench "$FOLDER/test/data/monaco.osrm.ramIndex" "$FOLDER/test/data/monaco.osrm.fileIndex" "$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench" - ./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench" - ./$BENCHMARKS_FOLDER/match-bench "./$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/match_ch.bench" - ./$BENCHMARKS_FOLDER/route-bench "./$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/route_mld.bench" - ./$BENCHMARKS_FOLDER/route-bench "./$FOLDER/test/data/ch/monaco.osrm" ch > "$RESULTS_FOLDER/route_ch.bench" - ./$BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench" - ./$BENCHMARKS_FOLDER/json-render-bench "./$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench" - ./$BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench" - ./$BENCHMARKS_FOLDER/rtree-bench "./$FOLDER/test/data/monaco.osrm.ramIndex" "./$FOLDER/test/data/monaco.osrm.fileIndex" "./$FOLDER/test/data/monaco.osrm.nbg_nodes" > "$RESULTS_FOLDER/rtree.bench" - - BINARIES_FOLDER="$FOLDER/build" - - cp ~/data.osm.pbf $FOLDER + cp -rf $OSM_PBF $FOLDER/data.osm.pbf + echo "Running osrm-extract" measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-extract -p $FOLDER/profiles/car.lua $FOLDER/data.osm.pbf" "$RESULTS_FOLDER/osrm_extract.bench" + echo "Running osrm-partition" measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-partition $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_partition.bench" + echo "Running osrm-customize" measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-customize $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_customize.bench" + echo "Running osrm-contract" measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench" - for BENCH in nearest table trip route match; do - ./$BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" mld ~/gps_traces.csv ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_mld.bench" || true - ./$BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" ch ~/gps_traces.csv ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_ch.bench" || true + for ALGORITHM in ch mld; do + for BENCH in nearest table trip route match; do + echo "Running random $BENCH $ALGORITHM" + START=$(date +%s.%N) + $BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" $ALGORITHM $GPS_TRACES ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_${ALGORITHM}.bench" 5 || true + END=$(date +%s.%N) + DIFF=$(echo "$END - $START" | bc) + echo "Took: ${DIFF}s" + done done for ALGORITHM in ch mld; do - $BINARIES_FOLDER/osrm-routed --algorithm $ALGORITHM $FOLDER/data.osrm & + $BINARIES_FOLDER/osrm-routed --algorithm $ALGORITHM $FOLDER/data.osrm > /dev/null 2>&1 & OSRM_ROUTED_PID=$! # wait for osrm-routed to start - if ! curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true"; then + if ! curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true" > /dev/null 2>&1; then echo "osrm-routed failed to start for algorithm $ALGORITHM" kill -9 $OSRM_ROUTED_PID continue fi for METHOD in route nearest trip table match; do - python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --num_requests 1000 > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench + echo "Running e2e benchmark for $METHOD $ALGORITHM" + START=$(date +%s.%N) + python3 $SCRIPTS_FOLDER/scripts/ci/e2e_benchmark.py --host http://localhost:5000 --method $METHOD --iterations 5 --num_requests 1000 --gps_traces_file_path $GPS_TRACES > $RESULTS_FOLDER/e2e_${METHOD}_${ALGORITHM}.bench + END=$(date +%s.%N) + DIFF=$(echo "$END - $START" | bc) + echo "Took: ${DIFF}s" done kill -9 $OSRM_ROUTED_PID done } -run_benchmarks_for_folder $1 "${1}_results" $2 -run_benchmarks_for_folder $2 "${2}_results" $2 +run_benchmarks_for_folder diff --git a/src/benchmarks/bench.cpp b/src/benchmarks/bench.cpp index ef53c398d..b1d88aa0e 100644 --- a/src/benchmarks/bench.cpp +++ b/src/benchmarks/bench.cpp @@ -45,8 +45,12 @@ class GPSTraces std::vector coordinates; mutable std::mt19937 gen; + int seed; + public: - GPSTraces(int seed) : gen(std::random_device{}()) { gen.seed(seed); } + GPSTraces(int seed) : gen(std::random_device{}()), seed(seed) { gen.seed(seed); } + + void resetSeed() const { gen.seed(seed); } bool readCSV(const std::string &filename) { @@ -101,75 +105,200 @@ class GPSTraces return coordinates[dis(gen)]; } - const std::vector &getRandomTrace() const + std::vector getRandomTrace() const { std::uniform_int_distribution<> dis(0, trackIDs.size() - 1); auto it = trackIDs.begin(); std::advance(it, dis(gen)); - return traces.at(*it); + + const auto &trace = traces.at(*it); + + std::uniform_int_distribution<> length_dis(50, 100); + size_t length = length_dis(gen); + if (trace.size() <= length + 1) + { + return trace; + } + + std::uniform_int_distribution<> start_dis(0, trace.size() - length - 1); + size_t start_index = start_dis(gen); + + return std::vector(trace.begin() + start_index, + trace.begin() + start_index + length); } }; +// Struct to hold confidence interval data +struct ConfidenceInterval +{ + double mean; + double confidence; + double min; +}; + +// Helper function to calculate the bootstrap confidence interval +ConfidenceInterval confidenceInterval(const std::vector &data, + int num_samples = 1000, + double confidence_level = 0.95) +{ + std::vector means; + std::default_random_engine generator; + std::uniform_int_distribution distribution(0, data.size() - 1); + + for (int i = 0; i < num_samples; ++i) + { + std::vector sample; + for (size_t j = 0; j < data.size(); ++j) + { + sample.push_back(data[distribution(generator)]); + } + double sample_mean = std::accumulate(sample.begin(), sample.end(), 0.0) / sample.size(); + means.push_back(sample_mean); + } + + std::sort(means.begin(), means.end()); + double lower_bound = means[(int)((1 - confidence_level) / 2 * num_samples)]; + double upper_bound = means[(int)((1 + confidence_level) / 2 * num_samples)]; + double mean = std::accumulate(means.begin(), means.end(), 0.0) / means.size(); + + ConfidenceInterval ci = { + mean, (upper_bound - lower_bound) / 2, *std::min_element(data.begin(), data.end())}; + return ci; +} + class Statistics { public: - void push(double timeMs) - { - times.push_back(timeMs); - sorted = false; - } + explicit Statistics(int iterations) : times(iterations) {} - double mean() { return sum() / times.size(); } + void push(double timeMs, int iteration) { times[iteration].push_back(timeMs); } - double sum() + ConfidenceInterval mean() { - double sum = 0; - for (auto time : times) + std::vector means; + means.reserve(times.size()); + for (const auto &iter_times : times) { - sum += time; + means.push_back(std::accumulate(iter_times.begin(), iter_times.end(), 0.0) / + iter_times.size()); } - return sum; + return confidenceInterval(means); } - double min() { return *std::min_element(times.begin(), times.end()); } - - double max() { return *std::max_element(times.begin(), times.end()); } - - double percentile(double p) + ConfidenceInterval total() { - const auto × = getTimes(); - return times[static_cast(p * times.size())]; + std::vector sums; + sums.reserve(times.size()); + for (const auto &iter_times : times) + { + sums.push_back(std::accumulate(iter_times.begin(), iter_times.end(), 0.0)); + } + return confidenceInterval(sums); + } + + ConfidenceInterval min() + { + std::vector mins; + mins.reserve(times.size()); + for (const auto &iter_times : times) + { + mins.push_back(*std::min_element(iter_times.begin(), iter_times.end())); + } + return confidenceInterval(mins); + } + + ConfidenceInterval max() + { + std::vector maxs; + maxs.reserve(times.size()); + for (const auto &iter_times : times) + { + maxs.push_back(*std::max_element(iter_times.begin(), iter_times.end())); + } + return confidenceInterval(maxs); + } + + ConfidenceInterval percentile(double p) + { + std::vector percentiles; + percentiles.reserve(times.size()); + for (const auto &iter_times : times) + { + auto sorted_times = iter_times; + std::sort(sorted_times.begin(), sorted_times.end()); + percentiles.push_back(sorted_times[static_cast(p * sorted_times.size())]); + } + return confidenceInterval(percentiles); + } + + ConfidenceInterval ops_per_sec() + { + std::vector ops; + ops.reserve(times.size()); + for (const auto &iter_times : times) + { + double total_time = std::accumulate(iter_times.begin(), iter_times.end(), 0.0) / 1000.0; + ops.push_back(iter_times.size() / total_time); + } + return confidenceInterval(ops); } private: - std::vector getTimes() - { - if (!sorted) - { - std::sort(times.begin(), times.end()); - sorted = true; - } - return times; - } - - std::vector times; - - bool sorted = false; + // vector of times for each iteration + std::vector> times; }; std::ostream &operator<<(std::ostream &os, Statistics &statistics) { os << std::fixed << std::setprecision(2); - os << "total: " << statistics.sum() << "ms" << std::endl; - os << "avg: " << statistics.mean() << "ms" << std::endl; - os << "min: " << statistics.min() << "ms" << std::endl; - os << "max: " << statistics.max() << "ms" << std::endl; - os << "p99: " << statistics.percentile(0.99) << "ms" << std::endl; + + ConfidenceInterval mean_ci = statistics.mean(); + ConfidenceInterval total_ci = statistics.total(); + ConfidenceInterval min_ci = statistics.min(); + ConfidenceInterval max_ci = statistics.max(); + ConfidenceInterval p99_ci = statistics.percentile(0.99); + ConfidenceInterval ops_ci = statistics.ops_per_sec(); + + os << "ops: " << ops_ci.mean << " ± " << ops_ci.confidence << " ops/s. " + << "best: " << ops_ci.min << "ops/s." << std::endl; + os << "total: " << total_ci.mean << " ± " << total_ci.confidence << "ms. " + << "best: " << total_ci.min << "ms." << std::endl; + os << "avg: " << mean_ci.mean << " ± " << mean_ci.confidence << "ms" << std::endl; + os << "min: " << min_ci.mean << " ± " << min_ci.confidence << "ms" << std::endl; + os << "max: " << max_ci.mean << " ± " << max_ci.confidence << "ms" << std::endl; + os << "p99: " << p99_ci.mean << " ± " << p99_ci.confidence << "ms" << std::endl; + return os; } -void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) +template +void runBenchmarks(const std::vector &benchmarks, + int iterations, + int opsPerIteration, + const OSRM &osrm, + const GPSTraces &gpsTraces, + const BenchmarkBody &benchmarkBody) { + for (const auto &benchmark : benchmarks) + { + Statistics statistics{iterations}; + for (int iteration = 0; iteration < iterations; ++iteration) + { + gpsTraces.resetSeed(); + + for (int i = 0; i < opsPerIteration; ++i) + { + benchmarkBody(iteration, benchmark, osrm, gpsTraces, statistics); + } + } + std::cout << benchmark.name << std::endl; + std::cout << statistics << std::endl; + } +} + +void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations) +{ + struct Benchmark { std::string name; @@ -179,114 +308,83 @@ void runRouteBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) std::optional alternatives = std::nullopt; std::optional radius = std::nullopt; }; - - auto run_benchmark = [&](const Benchmark &benchmark) - { - Statistics statistics; - - auto NUM = 10000; - for (int i = 0; i < NUM; ++i) - { - RouteParameters params; - params.overview = benchmark.overview; - params.steps = benchmark.steps; - - for (size_t i = 0; i < benchmark.coordinates; ++i) - { - params.coordinates.push_back(gpsTraces.getRandomCoordinate()); - } - - if (benchmark.alternatives) - { - params.alternatives = *benchmark.alternatives; - } - - if (benchmark.radius) - { - params.radiuses = std::vector>( - params.coordinates.size(), boost::make_optional(*benchmark.radius)); - } - - engine::api::ResultT result = json::Object(); - TIMER_START(routes); - const auto rc = osrm.Route(params, result); - TIMER_STOP(routes); - - statistics.push(TIMER_MSEC(routes)); - - auto &json_result = std::get(result); - if (rc != Status::Ok || json_result.values.find("routes") == json_result.values.end()) - { - auto code = std::get(json_result.values["code"]).value; - if (code != "NoSegment" && code != "NoRoute") - { - throw std::runtime_error{"Couldn't route: " + code}; - } - } - } - std::cout << benchmark.name << std::endl; - std::cout << statistics << std::endl; - }; - std::vector benchmarks = { - {"10000 routes, 3 coordinates, no alternatives, overview=full, steps=true", + {"1000 routes, 3 coordinates, no alternatives, overview=full, steps=true", 3, RouteParameters::OverviewType::Full, true, std::nullopt}, - {"10000 routes, 2 coordinates, no alternatives, overview=full, steps=true", - 2, - RouteParameters::OverviewType::Full, - true, - std::nullopt}, - {"10000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true", + {"1000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true", 2, RouteParameters::OverviewType::Full, true, 3}, - {"10000 routes, 3 coordinates, no alternatives, overview=false, steps=false", + {"1000 routes, 3 coordinates, no alternatives, overview=false, steps=false", 3, RouteParameters::OverviewType::False, false, std::nullopt}, - {"10000 routes, 2 coordinates, no alternatives, overview=false, steps=false", + {"1000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false", 2, RouteParameters::OverviewType::False, false, - std::nullopt}, - {"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false", - 2, - RouteParameters::OverviewType::False, - false, - 3}, - {"10000 routes, 3 coordinates, no alternatives, overview=false, steps=false, radius=750", - 3, - RouteParameters::OverviewType::False, - false, - std::nullopt, - 750}, - {"10000 routes, 2 coordinates, no alternatives, overview=false, steps=false, radius=750", - 2, - RouteParameters::OverviewType::False, - false, - std::nullopt, - 750}, - {"10000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false, radius=750", - 2, - RouteParameters::OverviewType::False, - false, - 3, - 750} + 3}}; - }; + runBenchmarks(benchmarks, + iterations, + 1000, + osrm, + gpsTraces, + [](int iteration, + const Benchmark &benchmark, + const OSRM &osrm, + const GPSTraces &gpsTraces, + Statistics &statistics) + { + RouteParameters params; + params.overview = benchmark.overview; + params.steps = benchmark.steps; - for (const auto &benchmark : benchmarks) - { - run_benchmark(benchmark); - } + for (size_t i = 0; i < benchmark.coordinates; ++i) + { + params.coordinates.push_back(gpsTraces.getRandomCoordinate()); + } + + if (benchmark.alternatives) + { + params.alternatives = *benchmark.alternatives; + } + + if (benchmark.radius) + { + params.radiuses = std::vector>( + params.coordinates.size(), boost::make_optional(*benchmark.radius)); + } + + engine::api::ResultT result = json::Object(); + TIMER_START(routes); + const auto rc = osrm.Route(params, result); + TIMER_STOP(routes); + + auto &json_result = std::get(result); + if (rc != Status::Ok || + json_result.values.find("routes") == json_result.values.end()) + { + auto code = std::get(json_result.values["code"]).value; + if (code != "NoSegment" && code != "NoRoute") + { + throw std::runtime_error{"Couldn't route: " + code}; + } + } + else + { + + statistics.push(TIMER_MSEC(routes), iteration); + } + }); } -void runMatchBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) +void runMatchBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations) { struct Benchmark { @@ -294,59 +392,56 @@ void runMatchBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) std::optional radius = std::nullopt; }; - auto run_benchmark = [&](const Benchmark &benchmark) - { - Statistics statistics; + std::vector benchmarks = {{"500 matches, default radius"}, + {"500 matches, radius=10", 10}, + {"500 matches, radius=20", 20}}; - auto NUM = 1000; - for (int i = 0; i < NUM; ++i) - { - engine::api::ResultT result = json::Object(); + runBenchmarks(benchmarks, + iterations, + 500, + osrm, + gpsTraces, + [](int iteration, + const Benchmark &benchmark, + const OSRM &osrm, + const GPSTraces &gpsTraces, + Statistics &statistics) + { + engine::api::ResultT result = json::Object(); - engine::api::MatchParameters params; - params.coordinates = gpsTraces.getRandomTrace(); - params.radiuses = {}; - if (benchmark.radius) - { - for (size_t index = 0; index < params.coordinates.size(); ++index) - { - params.radiuses.emplace_back(*benchmark.radius); - } - } + engine::api::MatchParameters params; + params.coordinates = gpsTraces.getRandomTrace(); + params.radiuses = {}; + if (benchmark.radius) + { + for (size_t index = 0; index < params.coordinates.size(); ++index) + { + params.radiuses.emplace_back(*benchmark.radius); + } + } - TIMER_START(match); - const auto rc = osrm.Match(params, result); - TIMER_STOP(match); + TIMER_START(match); + const auto rc = osrm.Match(params, result); + TIMER_STOP(match); - statistics.push(TIMER_MSEC(match)); - - auto &json_result = std::get(result); - if (rc != Status::Ok || - json_result.values.find("matchings") == json_result.values.end()) - { - auto code = std::get(json_result.values["code"]).value; - if (code != "NoSegment" && code != "NoMatch") - { - throw std::runtime_error{"Couldn't route: " + code}; - } - } - } - - std::cout << benchmark.name << std::endl; - std::cout << statistics << std::endl; - }; - - std::vector benchmarks = {{"1000 matches, default radius"}, - {"1000 matches, radius=10", 10}, - {"1000 matches, radius=20", 20}}; - - for (const auto &benchmark : benchmarks) - { - run_benchmark(benchmark); - } + auto &json_result = std::get(result); + if (rc != Status::Ok || + json_result.values.find("matchings") == json_result.values.end()) + { + auto code = std::get(json_result.values["code"]).value; + if (code != "NoSegment" && code != "NoMatch") + { + throw std::runtime_error{"Couldn't route: " + code}; + } + } + else + { + statistics.push(TIMER_MSEC(match), iteration); + } + }); } -void runNearestBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) +void runNearestBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations) { struct Benchmark { @@ -354,54 +449,52 @@ void runNearestBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) std::optional number_of_results = std::nullopt; }; - auto run_benchmark = [&](const Benchmark &benchmark) - { - Statistics statistics; - auto NUM = 10000; - for (int i = 0; i < NUM; ++i) - { - engine::api::ResultT result = json::Object(); - NearestParameters params; - params.coordinates.push_back(gpsTraces.getRandomCoordinate()); - - if (benchmark.number_of_results) - { - params.number_of_results = *benchmark.number_of_results; - } - - TIMER_START(nearest); - const auto rc = osrm.Nearest(params, result); - TIMER_STOP(nearest); - - statistics.push(TIMER_MSEC(nearest)); - - auto &json_result = std::get(result); - if (rc != Status::Ok || - json_result.values.find("waypoints") == json_result.values.end()) - { - auto code = std::get(json_result.values["code"]).value; - if (code != "NoSegment") - { - throw std::runtime_error{"Couldn't find nearest point"}; - } - } - } - - std::cout << benchmark.name << std::endl; - std::cout << statistics << std::endl; - }; - std::vector benchmarks = {{"10000 nearest, number_of_results=1", 1}, {"10000 nearest, number_of_results=5", 5}, {"10000 nearest, number_of_results=10", 10}}; - for (const auto &benchmark : benchmarks) - { - run_benchmark(benchmark); - } + runBenchmarks(benchmarks, + iterations, + 10000, + osrm, + gpsTraces, + [](int iteration, + const Benchmark &benchmark, + const OSRM &osrm, + const GPSTraces &gpsTraces, + Statistics &statistics) + { + engine::api::ResultT result = json::Object(); + NearestParameters params; + params.coordinates.push_back(gpsTraces.getRandomCoordinate()); + + if (benchmark.number_of_results) + { + params.number_of_results = *benchmark.number_of_results; + } + + TIMER_START(nearest); + const auto rc = osrm.Nearest(params, result); + TIMER_STOP(nearest); + + auto &json_result = std::get(result); + if (rc != Status::Ok || + json_result.values.find("waypoints") == json_result.values.end()) + { + auto code = std::get(json_result.values["code"]).value; + if (code != "NoSegment") + { + throw std::runtime_error{"Couldn't find nearest point"}; + } + } + else + { + statistics.push(TIMER_MSEC(nearest), iteration); + } + }); } -void runTripBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) +void runTripBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations) { struct Benchmark { @@ -409,54 +502,52 @@ void runTripBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) size_t coordinates; }; - auto run_benchmark = [&](const Benchmark &benchmark) - { - Statistics statistics; - auto NUM = 1000; - for (int i = 0; i < NUM; ++i) - { - engine::api::ResultT result = json::Object(); - TripParameters params; - params.roundtrip = true; - - for (size_t i = 0; i < benchmark.coordinates; ++i) - { - params.coordinates.push_back(gpsTraces.getRandomCoordinate()); - } - - TIMER_START(trip); - const auto rc = osrm.Trip(params, result); - TIMER_STOP(trip); - - statistics.push(TIMER_MSEC(trip)); - - auto &json_result = std::get(result); - if (rc != Status::Ok || json_result.values.find("trips") == json_result.values.end()) - { - auto code = std::get(json_result.values["code"]).value; - if (code != "NoSegment") - { - throw std::runtime_error{"Couldn't find trip"}; - } - } - } - - std::cout << benchmark.name << std::endl; - std::cout << statistics << std::endl; - }; - std::vector benchmarks = { - {"1000 trips, 3 coordinates", 3}, - {"1000 trips, 4 coordinates", 4}, - {"1000 trips, 5 coordinates", 5}, + {"250 trips, 3 coordinates", 3}, + {"250 trips, 5 coordinates", 5}, }; - for (const auto &benchmark : benchmarks) - { - run_benchmark(benchmark); - } + runBenchmarks(benchmarks, + iterations, + 250, + osrm, + gpsTraces, + [](int iteration, + const Benchmark &benchmark, + const OSRM &osrm, + const GPSTraces &gpsTraces, + Statistics &statistics) + { + engine::api::ResultT result = json::Object(); + TripParameters params; + params.roundtrip = true; + + for (size_t i = 0; i < benchmark.coordinates; ++i) + { + params.coordinates.push_back(gpsTraces.getRandomCoordinate()); + } + + TIMER_START(trip); + const auto rc = osrm.Trip(params, result); + TIMER_STOP(trip); + + auto &json_result = std::get(result); + if (rc != Status::Ok || + json_result.values.find("trips") == json_result.values.end()) + { + auto code = std::get(json_result.values["code"]).value; + if (code != "NoSegment") + { + throw std::runtime_error{"Couldn't find trip"}; + } + } + else + { + statistics.push(TIMER_MSEC(trip), iteration); + } + }); } -void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) +void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces, int iterations) { struct Benchmark { @@ -464,51 +555,46 @@ void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) size_t coordinates; }; - auto run_benchmark = [&](const Benchmark &benchmark) - { - Statistics statistics; - auto NUM = 250; - for (int i = 0; i < NUM; ++i) - { - engine::api::ResultT result = json::Object(); - TableParameters params; - - for (size_t i = 0; i < benchmark.coordinates; ++i) - { - params.coordinates.push_back(gpsTraces.getRandomCoordinate()); - } - - TIMER_START(table); - const auto rc = osrm.Table(params, result); - TIMER_STOP(table); - - statistics.push(TIMER_MSEC(table)); - - auto &json_result = std::get(result); - if (rc != Status::Ok || - json_result.values.find("durations") == json_result.values.end()) - { - auto code = std::get(json_result.values["code"]).value; - if (code != "NoSegment") - { - throw std::runtime_error{"Couldn't compute table"}; - } - } - } - - std::cout << benchmark.name << std::endl; - std::cout << statistics << std::endl; - }; - std::vector benchmarks = {{"250 tables, 3 coordinates", 3}, {"250 tables, 25 coordinates", 25}, - {"250 tables, 50 coordinates", 50}, - {"250 tables, 100 coordinates", 100}}; + {"250 tables, 50 coordinates", 50}}; - for (const auto &benchmark : benchmarks) - { - run_benchmark(benchmark); - } + runBenchmarks(benchmarks, + iterations, + 250, + osrm, + gpsTraces, + [](int iteration, + const Benchmark &benchmark, + const OSRM &osrm, + const GPSTraces &gpsTraces, + Statistics &statistics) + { + engine::api::ResultT result = json::Object(); + TableParameters params; + + for (size_t i = 0; i < benchmark.coordinates; ++i) + { + params.coordinates.push_back(gpsTraces.getRandomCoordinate()); + } + + TIMER_START(table); + const auto rc = osrm.Table(params, result); + TIMER_STOP(table); + + statistics.push(TIMER_MSEC(table), iteration); + + auto &json_result = std::get(result); + if (rc != Status::Ok || + json_result.values.find("durations") == json_result.values.end()) + { + auto code = std::get(json_result.values["code"]).value; + if (code != "NoSegment") + { + throw std::runtime_error{"Couldn't compute table"}; + } + } + }); } } // namespace @@ -516,11 +602,11 @@ void runTableBenchmark(const OSRM &osrm, const GPSTraces &gpsTraces) int main(int argc, const char *argv[]) try { - if (argc < 5) + if (argc < 6) { - std::cerr - << "Usage: " << argv[0] - << " data.osrm \n"; + std::cerr << "Usage: " << argv[0] + << " data.osrm " + " \n"; return EXIT_FAILURE; } @@ -537,27 +623,29 @@ try GPSTraces gpsTraces{42}; gpsTraces.readCSV(argv[3]); + int iterations = std::stoi(argv[5]); + const auto benchmarkToRun = std::string{argv[4]}; if (benchmarkToRun == "route") { - runRouteBenchmark(osrm, gpsTraces); + runRouteBenchmark(osrm, gpsTraces, iterations); } else if (benchmarkToRun == "match") { - runMatchBenchmark(osrm, gpsTraces); + runMatchBenchmark(osrm, gpsTraces, iterations); } else if (benchmarkToRun == "nearest") { - runNearestBenchmark(osrm, gpsTraces); + runNearestBenchmark(osrm, gpsTraces, iterations); } else if (benchmarkToRun == "trip") { - runTripBenchmark(osrm, gpsTraces); + runTripBenchmark(osrm, gpsTraces, iterations); } else if (benchmarkToRun == "table") { - runTableBenchmark(osrm, gpsTraces); + runTableBenchmark(osrm, gpsTraces, iterations); } else { diff --git a/src/benchmarks/match.cpp b/src/benchmarks/match.cpp index 47a9b12b4..2e917c7a9 100644 --- a/src/benchmarks/match.cpp +++ b/src/benchmarks/match.cpp @@ -254,7 +254,7 @@ try << std::endl; }; - for (auto radius : std::vector>{std::nullopt, 5.0, 10.0, 15.0, 30.0}) + for (auto radius : std::vector>{std::nullopt, 10.0}) { run_benchmark(radius); } diff --git a/src/benchmarks/route.cpp b/src/benchmarks/route.cpp index def90c175..e63a1039e 100644 --- a/src/benchmarks/route.cpp +++ b/src/benchmarks/route.cpp @@ -96,12 +96,6 @@ try RouteParameters::OverviewType::Full, true, std::nullopt}, - {"1000 routes, 2 coordinates, no alternatives, overview=full, steps=true", - {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, - {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, - RouteParameters::OverviewType::Full, - true, - std::nullopt}, {"1000 routes, 2 coordinates, 3 alternatives, overview=full, steps=true", {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, @@ -115,40 +109,12 @@ try RouteParameters::OverviewType::False, false, std::nullopt}, - {"1000 routes, 2 coordinates, no alternatives, overview=false, steps=false", - {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, - {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, - RouteParameters::OverviewType::False, - false, - std::nullopt}, {"1000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false", {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, RouteParameters::OverviewType::False, false, 3}, - {"1000 routes, 3 coordinates, no alternatives, overview=false, steps=false, radius=750", - {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, - {FloatLongitude{7.421844922513342}, FloatLatitude{43.73690777888953}}, - {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, - RouteParameters::OverviewType::False, - false, - std::nullopt, - 750}, - {"1000 routes, 2 coordinates, no alternatives, overview=false, steps=false, radius=750", - {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, - {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, - RouteParameters::OverviewType::False, - false, - std::nullopt, - 750}, - {"1000 routes, 2 coordinates, 3 alternatives, overview=false, steps=false, radius=750", - {{FloatLongitude{7.437602352715465}, FloatLatitude{43.75030522209604}}, - {FloatLongitude{7.412303912230966}, FloatLatitude{43.72851046529198}}}, - RouteParameters::OverviewType::False, - false, - 3, - 750} }; From 3d01d96036d6094b4924258657eee8be4d957732 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Sat, 22 Jun 2024 08:50:18 +0200 Subject: [PATCH 15/17] Avoid reallocations in base64 encoding (#6951) --- CHANGELOG.md | 1 + include/engine/base64.hpp | 27 ++++++----- unit_tests/engine/base64.cpp | 94 ++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0abc407c..5e7ef6891 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - NodeJS: - CHANGED: Use node-api instead of NAN. [#6452](https://github.com/Project-OSRM/osrm-backend/pull/6452) - Misc: + - CHANGED: Avoid reallocations in base64 encoding. [#6951](https://github.com/Project-OSRM/osrm-backend/pull/6951) - CHANGED: Get rid of unused Boost dependencies. [#6960](https://github.com/Project-OSRM/osrm-backend/pull/6960) - CHANGED: Apply micro-optimisation for Table & Trip APIs. [#6949](https://github.com/Project-OSRM/osrm-backend/pull/6949) - CHANGED: Apply micro-optimisation for Route API. [#6948](https://github.com/Project-OSRM/osrm-backend/pull/6948) diff --git a/include/engine/base64.hpp b/include/engine/base64.hpp index d1fa8753c..042aa6d59 100644 --- a/include/engine/base64.hpp +++ b/include/engine/base64.hpp @@ -47,24 +47,29 @@ namespace engine // Encodes a chunk of memory to Base64. inline std::string encodeBase64(const unsigned char *first, std::size_t size) { - std::vector bytes{first, first + size}; - BOOST_ASSERT(!bytes.empty()); + BOOST_ASSERT(size > 0); - std::size_t bytes_to_pad{0}; + std::string encoded; + encoded.reserve(((size + 2) / 3) * 4); - while (bytes.size() % 3 != 0) + auto padding = (3 - size % 3) % 3; + + BOOST_ASSERT(padding == 0 || padding == 1 || padding == 2); + + for (auto itr = detail::Base64FromBinary(first); itr != detail::Base64FromBinary(first + size); + ++itr) { - bytes_to_pad += 1; - bytes.push_back(0); + encoded.push_back(*itr); } - BOOST_ASSERT(bytes_to_pad == 0 || bytes_to_pad == 1 || bytes_to_pad == 2); - BOOST_ASSERT_MSG(0 == bytes.size() % 3, "base64 input data size is not a multiple of 3"); + for (size_t index = 0; index < padding; ++index) + { + encoded.push_back('='); + } - std::string encoded{detail::Base64FromBinary{bytes.data()}, - detail::Base64FromBinary{bytes.data() + (bytes.size() - bytes_to_pad)}}; + BOOST_ASSERT(encoded.size() == (size + 2) / 3 * 4); - return encoded.append(bytes_to_pad, '='); + return encoded; } // C++11 standard 3.9.1/1: Plain char, signed char, and unsigned char are three distinct types diff --git a/unit_tests/engine/base64.cpp b/unit_tests/engine/base64.cpp index 3adb7db3e..60e5d73ca 100644 --- a/unit_tests/engine/base64.cpp +++ b/unit_tests/engine/base64.cpp @@ -74,4 +74,98 @@ BOOST_AUTO_TEST_CASE(hint_encoding_decoding_roundtrip_bytewise) reinterpret_cast(&decoded))); } +BOOST_AUTO_TEST_CASE(long_string_encoding) +{ + using namespace osrm::engine; + std::string long_string(1000, 'A'); // String of 1000 'A's + std::string encoded = encodeBase64(long_string); + BOOST_CHECK_EQUAL(decodeBase64(encoded), long_string); +} + +BOOST_AUTO_TEST_CASE(invalid_base64_decoding) +{ + using namespace osrm::engine; + BOOST_CHECK_THROW(decodeBase64("Invalid!"), std::exception); +} + +BOOST_AUTO_TEST_CASE(hint_serialization_size) +{ + using namespace osrm::engine; + using namespace osrm::util; + + const Coordinate coordinate; + const PhantomNode phantom; + const osrm::test::MockDataFacade facade{}; + + const SegmentHint hint{phantom, facade.GetCheckSum()}; + const auto base64 = hint.ToBase64(); + + BOOST_CHECK_EQUAL(base64.size(), 112); +} + +BOOST_AUTO_TEST_CASE(extended_roundtrip_tests) +{ + using namespace osrm::engine; + + std::vector test_strings = { + "Hello, World!", // Simple ASCII string + "1234567890", // Numeric string + "!@#$%^&*()_+", // Special characters + std::string(1000, 'A'), // Long repeating string + "¡Hola, mundo!", // Non-ASCII characters + "こんにちは、世界！", // Unicode characters + std::string("\x00\x01\x02\x03", 4), // Binary data + "a", // Single character + "ab", // Two characters + "abc", // Three characters (no padding in Base64) + std::string(190, 'x') // String that doesn't align with Base64 padding + }; + + for (const auto &test_str : test_strings) + { + std::string encoded = encodeBase64(test_str); + std::string decoded = decodeBase64(encoded); + BOOST_CHECK_EQUAL(decoded, test_str); + + // Additional checks + BOOST_CHECK(encoded.find_first_not_of( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=") == + std::string::npos); + if (test_str.length() % 3 != 0) + { + BOOST_CHECK(encoded.back() == '='); + } + } +} + +BOOST_AUTO_TEST_CASE(roundtrip_with_url_safe_chars) +{ + using namespace osrm::engine; + + std::string original = "Hello+World/Nothing?Is:Impossible"; + std::string encoded = encodeBase64(original); + + // Replace '+' with '-' and '/' with '_' + std::replace(encoded.begin(), encoded.end(), '+', '-'); + std::replace(encoded.begin(), encoded.end(), '/', '_'); + + std::string decoded = decodeBase64(encoded); + BOOST_CHECK_EQUAL(decoded, original); +} + +BOOST_AUTO_TEST_CASE(roundtrip_stress_test) +{ + using namespace osrm::engine; + + std::string test_str; + for (int i = 0; i < 1000; ++i) + { + test_str += static_cast(i % 256); + } + + std::string encoded = encodeBase64(test_str); + std::string decoded = decodeBase64(encoded); + BOOST_CHECK_EQUAL(decoded, test_str); +} + BOOST_AUTO_TEST_SUITE_END() From 7d72dfebf7750f3f64eba9596a2c8df1ddaf750a Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Sat, 22 Jun 2024 20:04:32 +0200 Subject: [PATCH 16/17] Optimise encodePolyline function (#6940) --- CHANGELOG.md | 1 + include/engine/polyline_compressor.hpp | 37 +++++---- src/engine/polyline_compressor.cpp | 39 +++------- unit_tests/engine/polyline_compressor.cpp | 92 +++++++++++++++++++++++ 4 files changed, 122 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e7ef6891..79a234ec9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - NodeJS: - CHANGED: Use node-api instead of NAN. [#6452](https://github.com/Project-OSRM/osrm-backend/pull/6452) - Misc: + - CHANGED: Optimise encodePolyline function. [#6940](https://github.com/Project-OSRM/osrm-backend/pull/6940) - CHANGED: Avoid reallocations in base64 encoding. [#6951](https://github.com/Project-OSRM/osrm-backend/pull/6951) - CHANGED: Get rid of unused Boost dependencies. [#6960](https://github.com/Project-OSRM/osrm-backend/pull/6960) - CHANGED: Apply micro-optimisation for Table & Trip APIs. [#6949](https://github.com/Project-OSRM/osrm-backend/pull/6949) diff --git a/include/engine/polyline_compressor.hpp b/include/engine/polyline_compressor.hpp index b2a505764..af58b0422 100644 --- a/include/engine/polyline_compressor.hpp +++ b/include/engine/polyline_compressor.hpp @@ -12,7 +12,7 @@ namespace osrm::engine { namespace detail { -std::string encode(std::vector &numbers); +void encode(int number_to_encode, std::string &output); std::int32_t decode_polyline_integer(std::string::const_iterator &first, std::string::const_iterator last); } // namespace detail @@ -30,27 +30,24 @@ std::string encodePolyline(CoordVectorForwardIter begin, CoordVectorForwardIter return {}; } - std::vector delta_numbers; - BOOST_ASSERT(size > 0); - delta_numbers.reserve((size - 1) * 2); + std::string output; + // just a guess that we will need ~4 bytes per coordinate to avoid reallocations + output.reserve(size * 4); + int current_lat = 0; int current_lon = 0; - std::for_each( - begin, - end, - [&delta_numbers, ¤t_lat, ¤t_lon, coordinate_to_polyline]( - const util::Coordinate loc) - { - const int lat_diff = - std::round(static_cast(loc.lat) * coordinate_to_polyline) - current_lat; - const int lon_diff = - std::round(static_cast(loc.lon) * coordinate_to_polyline) - current_lon; - delta_numbers.emplace_back(lat_diff); - delta_numbers.emplace_back(lon_diff); - current_lat += lat_diff; - current_lon += lon_diff; - }); - return detail::encode(delta_numbers); + for (auto it = begin; it != end; ++it) + { + const int lat_diff = + std::round(static_cast(it->lat) * coordinate_to_polyline) - current_lat; + const int lon_diff = + std::round(static_cast(it->lon) * coordinate_to_polyline) - current_lon; + detail::encode(lat_diff, output); + detail::encode(lon_diff, output); + current_lat += lat_diff; + current_lon += lon_diff; + } + return output; } // Decodes geometry from polyline format diff --git a/src/engine/polyline_compressor.cpp b/src/engine/polyline_compressor.cpp index 3d4500643..7435fed3f 100644 --- a/src/engine/polyline_compressor.cpp +++ b/src/engine/polyline_compressor.cpp @@ -10,9 +10,19 @@ namespace osrm::engine::detail // anonymous to keep TU local { -std::string encode(int number_to_encode) +void encode(int number_to_encode, std::string &output) { - std::string output; + if (number_to_encode < 0) + { + const unsigned binary = std::llabs(number_to_encode); + const unsigned twos = (~binary) + 1u; + const unsigned shl = twos << 1u; + number_to_encode = static_cast(~shl); + } + else + { + number_to_encode <<= 1u; + } while (number_to_encode >= 0x20) { const int next_value = (0x20 | (number_to_encode & 0x1f)) + 63; @@ -22,31 +32,6 @@ std::string encode(int number_to_encode) number_to_encode += 63; output += static_cast(number_to_encode); - return output; -} - -std::string encode(std::vector &numbers) -{ - std::string output; - for (auto &number : numbers) - { - if (number < 0) - { - const unsigned binary = std::llabs(number); - const unsigned twos = (~binary) + 1u; - const unsigned shl = twos << 1u; - number = static_cast(~shl); - } - else - { - number <<= 1u; - } - } - for (const int number : numbers) - { - output += encode(number); - } - return output; } // https://developers.google.com/maps/documentation/utilities/polylinealgorithm diff --git a/unit_tests/engine/polyline_compressor.cpp b/unit_tests/engine/polyline_compressor.cpp index fa05512a9..e2a9a5fd7 100644 --- a/unit_tests/engine/polyline_compressor.cpp +++ b/unit_tests/engine/polyline_compressor.cpp @@ -45,4 +45,96 @@ BOOST_AUTO_TEST_CASE(polyline6_test_case) decodePolyline<1000000>(encodePolyline<1000000>(coords.begin(), coords.end())).begin())); } +BOOST_AUTO_TEST_CASE(empty_polyline_test) +{ + using namespace osrm::engine; + using namespace osrm::util; + + std::vector empty_coords; + BOOST_CHECK_EQUAL(encodePolyline(empty_coords.begin(), empty_coords.end()), ""); + BOOST_CHECK(decodePolyline("").empty()); +} +BOOST_AUTO_TEST_CASE(polyline_single_point_test) +{ + using namespace osrm::engine; + using namespace osrm::util; + + const std::vector coords({{FixedLongitude{-122414000}, FixedLatitude{37776000}}}); + + const std::string encoded = encodePolyline(coords.begin(), coords.end()); + BOOST_CHECK_EQUAL(encoded, "_cqeFn~cjV"); + + const auto decoded = decodePolyline(encoded); + BOOST_CHECK_EQUAL(decoded.size(), 1); + BOOST_CHECK_EQUAL(decoded[0].lon, FixedLongitude{-122414000}); + BOOST_CHECK_EQUAL(decoded[0].lat, FixedLatitude{37776000}); +} + +BOOST_AUTO_TEST_CASE(polyline_multiple_points_test) +{ + using namespace osrm::engine; + using namespace osrm::util; + + const std::vector coords({{FixedLongitude{-122414000}, FixedLatitude{37776000}}, + {FixedLongitude{-122420000}, FixedLatitude{37779000}}, + {FixedLongitude{-122421000}, FixedLatitude{37780000}}}); + + const std::string encoded = encodePolyline(coords.begin(), coords.end()); + BOOST_CHECK_EQUAL(encoded, "_cqeFn~cjVwQnd@gEfE"); + + const auto decoded = decodePolyline(encoded); + BOOST_CHECK_EQUAL(decoded.size(), 3); + for (size_t i = 0; i < coords.size(); ++i) + { + BOOST_CHECK_EQUAL(decoded[i].lon, coords[i].lon); + BOOST_CHECK_EQUAL(decoded[i].lat, coords[i].lat); + } +} + +BOOST_AUTO_TEST_CASE(polyline_large_coordinate_difference_test) +{ + using namespace osrm::engine; + using namespace osrm::util; + + const std::vector coords({{FixedLongitude{-179000000}, FixedLatitude{-89000000}}, + {FixedLongitude{179000000}, FixedLatitude{89000000}}}); + + const std::string encoded = encodePolyline(coords.begin(), coords.end()); + BOOST_CHECK_EQUAL(encoded, "~xe~O~|oca@_sl}`@_{`hcA"); + + const auto decoded = decodePolyline(encoded); + BOOST_CHECK_EQUAL(decoded.size(), 2); + for (size_t i = 0; i < coords.size(); ++i) + { + BOOST_CHECK_EQUAL(decoded[i].lon, coords[i].lon); + BOOST_CHECK_EQUAL(decoded[i].lat, coords[i].lat); + } +} + +BOOST_AUTO_TEST_CASE(roundtrip) +{ + using namespace osrm::engine; + using namespace osrm::util; + + { + const auto encoded = "_chxEn`zvN\\\\]]"; + const auto decoded = decodePolyline(encoded); + const auto reencoded = encodePolyline(decoded.begin(), decoded.end()); + BOOST_CHECK_EQUAL(encoded, reencoded); + } + { + const auto encoded = + "gcneIpgxzRcDnBoBlEHzKjBbHlG`@`IkDxIiKhKoMaLwTwHeIqHuAyGXeB~Ew@fFjAtIzExF"; + const auto decoded = decodePolyline(encoded); + const auto reencoded = encodePolyline(decoded.begin(), decoded.end()); + BOOST_CHECK_EQUAL(encoded, reencoded); + } + { + const auto encoded = "_p~iF~ps|U_ulLnnqC_mqNvxq`@"; + const auto decoded = decodePolyline(encoded); + const auto reencoded = encodePolyline(decoded.begin(), decoded.end()); + BOOST_CHECK_EQUAL(encoded, reencoded); + } +} + BOOST_AUTO_TEST_SUITE_END() From 5a48ce85b3b8d521e556af23ad85663760b1ae19 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Tue, 25 Jun 2024 17:49:41 +0200 Subject: [PATCH 17/17] Use tmpfs for running benchmarks (#6966) --- .github/workflows/osrm-backend.yml | 26 ++++++++++++++++--- scripts/ci/run_benchmarks.sh | 2 +- .../data}/portugal_to_korea.json | 0 3 files changed, 24 insertions(+), 4 deletions(-) rename {src/benchmarks => test/data}/portugal_to_korea.json (100%) diff --git a/.github/workflows/osrm-backend.yml b/.github/workflows/osrm-backend.yml index b743ee942..10df684c4 100644 --- a/.github/workflows/osrm-backend.yml +++ b/.github/workflows/osrm-backend.yml @@ -723,14 +723,34 @@ jobs: make -j$(nproc) benchmarks cd .. make -C test/data + # we run benchmarks in tmpfs to avoid impact of disk IO + - name: Create folder for tmpfs + run: mkdir -p /opt/benchmarks - name: Run PR Benchmarks run: | - ./pr/scripts/ci/run_benchmarks.sh -f $(pwd)/pr -r $(pwd)/pr_results -s $(pwd)/pr -b $(pwd)/pr/build -o ~/data.osm.pbf -g ~/gps_traces.csv + sudo mount -t tmpfs -o size=4g none /opt/benchmarks + cp -rf pr/build /opt/benchmarks/build + mkdir -p /opt/benchmarks/test + cp -rf pr/test/data /opt/benchmarks/test/data + cp -rf pr/profiles /opt/benchmarks/profiles + + ./pr/scripts/ci/run_benchmarks.sh -f /opt/benchmarks -r $(pwd)/pr_results -s $(pwd)/pr -b /opt/benchmarks/build -o ~/data.osm.pbf -g ~/gps_traces.csv + sudo umount /opt/benchmarks - name: Run Base Benchmarks run: | - # we intentionally use scripts from PR branch to be able to update them and see results in the same PR - ./pr/scripts/ci/run_benchmarks.sh -f $(pwd)/base -r $(pwd)/base_results -s $(pwd)/pr -b $(pwd)/base/build -o ~/data.osm.pbf -g ~/gps_traces.csv + sudo mount -t tmpfs -o size=4g none /opt/benchmarks + cp -rf base/build /opt/benchmarks/build + mkdir -p /opt/benchmarks/test + cp -rf base/test/data /opt/benchmarks/test/data + cp -rf base/profiles /opt/benchmarks/profiles + # TODO: remove it when base branch will have this file at needed location + if [ ! -f /opt/benchmarks/test/data/portugal_to_korea.json ]; then + cp base/src/benchmarks/portugal_to_korea.json /opt/benchmarks/test/data/portugal_to_korea.json + fi + # we intentionally use scripts from PR branch to be able to update them and see results in the same PR + ./pr/scripts/ci/run_benchmarks.sh -f /opt/benchmarks -r $(pwd)/base_results -s $(pwd)/pr -b /opt/benchmarks/build -o ~/data.osm.pbf -g ~/gps_traces.csv + sudo umount /opt/benchmarks - name: Post Benchmark Results run: | python3 pr/scripts/ci/post_benchmark_results.py base_results pr_results diff --git a/scripts/ci/run_benchmarks.sh b/scripts/ci/run_benchmarks.sh index a0f32ece5..0d0324d13 100755 --- a/scripts/ci/run_benchmarks.sh +++ b/scripts/ci/run_benchmarks.sh @@ -64,7 +64,7 @@ function run_benchmarks_for_folder { echo "Running alias" $BENCHMARKS_FOLDER/alias-bench > "$RESULTS_FOLDER/alias.bench" echo "Running json-render-bench" - $BENCHMARKS_FOLDER/json-render-bench "$FOLDER/src/benchmarks/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench" + $BENCHMARKS_FOLDER/json-render-bench "$FOLDER/test/data/portugal_to_korea.json" > "$RESULTS_FOLDER/json-render.bench" echo "Running packedvector-bench" $BENCHMARKS_FOLDER/packedvector-bench > "$RESULTS_FOLDER/packedvector.bench" echo "Running rtree-bench" diff --git a/src/benchmarks/portugal_to_korea.json b/test/data/portugal_to_korea.json similarity index 100% rename from src/benchmarks/portugal_to_korea.json rename to test/data/portugal_to_korea.json