From 17f8339aa6a7361249d7fa866259bc66db24f764 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 11 Mar 2017 01:07:52 +0000 Subject: [PATCH] Split out segment data update function --- src/updater/updater.cpp | 404 ++++++++++++++++++++-------------------- 1 file changed, 199 insertions(+), 205 deletions(-) diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index 6a411e512..ae9ac41e0 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -42,6 +42,10 @@ #include #include +namespace osrm +{ +namespace updater +{ namespace { @@ -51,15 +55,8 @@ template inline bool is_aligned(const void *pointer) return reinterpret_cast(pointer) % alignof(T) == 0; } -} // anon ns - -namespace osrm -{ -namespace updater -{ - // Returns duration in deci-seconds -inline EdgeWeight ConvertToDuration(double distance_in_meters, double speed_in_kmh) +inline EdgeWeight convertToDuration(double distance_in_meters, double speed_in_kmh) { if (speed_in_kmh <= 0.) return MAXIMAL_EDGE_DURATION; @@ -69,7 +66,7 @@ inline EdgeWeight ConvertToDuration(double distance_in_meters, double speed_in_k return std::max(1, static_cast(std::round(duration * 10.))); } -inline EdgeWeight ConvertToWeight(double weight, double weight_multiplier, EdgeWeight duration) +inline EdgeWeight convertToWeight(double weight, double weight_multiplier, EdgeWeight duration) { if (std::isfinite(weight)) return std::round(weight * weight_multiplier); @@ -79,7 +76,7 @@ inline EdgeWeight ConvertToWeight(double weight, double weight_multiplier, EdgeW } // Returns updated edge weight -void GetNewWeight(const UpdaterConfig &config, +void getNewWeight(const UpdaterConfig &config, const double weight_multiplier, const SpeedSource &value, const double &segment_length, @@ -90,10 +87,10 @@ void GetNewWeight(const UpdaterConfig &config, EdgeWeight &new_segment_duration) { // Update the edge duration as distance/speed - new_segment_duration = ConvertToDuration(segment_length, value.speed); + new_segment_duration = convertToDuration(segment_length, value.speed); // Update the edge weight or fallback to the new edge duration - new_segment_weight = ConvertToWeight(value.weight, weight_multiplier, new_segment_duration); + new_segment_weight = convertToWeight(value.weight, weight_multiplier, new_segment_duration); // The check here is enabled by the `--edge-weight-updates-over-factor` flag it logs a warning // if the new duration exceeds a heuristic of what a reasonable duration update is @@ -117,7 +114,7 @@ void GetNewWeight(const UpdaterConfig &config, } #if !defined(NDEBUG) -void CheckWeightsConsistency( +void checkWeightsConsistency( const UpdaterConfig &config, const std::vector &edge_based_edge_list) { @@ -162,6 +159,188 @@ void CheckWeightsConsistency( } #endif +auto mmapFile(const std::string &filename, boost::interprocess::mode_t mode) +{ + using boost::interprocess::file_mapping; + using boost::interprocess::mapped_region; + + try + { + const file_mapping mapping{filename.c_str(), mode}; + mapped_region region{mapping, mode}; + region.advise(mapped_region::advice_sequential); + return region; + } + catch (const std::exception &e) + { + util::Log(logERROR) << "Error while trying to mmap " + filename + ": " + e.what(); + throw; + } +} + +void updaterSegmentData(const UpdaterConfig &config, + const extractor::ProfileProperties &profile_properties, + const SegmentLookupTable &segment_speed_lookup) +{ + auto weight_multiplier = profile_properties.GetWeightMultiplier(); + std::vector internal_to_external_node_map; + extractor::SegmentDataContainer segment_data; + + const auto load_internal_to_external_node_map = [&] { + storage::io::FileReader nodes_file(config.node_based_graph_path, + storage::io::FileReader::HasNoFingerprint); + + nodes_file.DeserializeVector(internal_to_external_node_map); + }; + + const auto load_geometries = [&] { extractor::io::read(config.geometry_path, segment_data); }; + + // Folds all our actions into independently concurrently executing lambdas + tbb::parallel_invoke(load_internal_to_external_node_map, load_geometries); + + // Here, we have to update the compressed geometry weights + // First, we need the external-to-internal node lookup table + + // Now, we iterate over all the segments stored in the StaticRTree, updating + // the packed geometry weights in the `.geometries` file (note: we do not + // update the RTree itself, we just use the leaf nodes to iterate over all segments) + using LeafNode = util::StaticRTree::LeafNode; + + using boost::interprocess::mapped_region; + + auto region = mmapFile(config.rtree_leaf_path.c_str(), boost::interprocess::read_only); + region.advise(mapped_region::advice_willneed); + BOOST_ASSERT(is_aligned(region.get_address())); + + const auto bytes = region.get_size(); + const auto first = static_cast(region.get_address()); + const auto last = first + (bytes / sizeof(LeafNode)); + + // vector to count used speeds for logging + // size offset by one since index 0 is used for speeds not from external file + using counters_type = std::vector; + std::size_t num_counters = config.segment_speed_lookup_paths.size() + 1; + tbb::enumerable_thread_specific segment_speeds_counters( + counters_type(num_counters, 0)); + const constexpr auto LUA_SOURCE = 0; + + tbb::parallel_for_each(first, last, [&](const LeafNode ¤t_node) { + auto &counters = segment_speeds_counters.local(); + for (size_t i = 0; i < current_node.object_count; i++) + { + const auto &leaf_object = current_node.objects[i]; + + const auto geometry_id = leaf_object.packed_geometry_id; + auto nodes_range = segment_data.GetForwardGeometry(geometry_id); + + const auto segment_offset = leaf_object.fwd_segment_position; + const std::uint32_t u_index = segment_offset; + const std::uint32_t v_index = segment_offset + 1; + + BOOST_ASSERT(u_index < nodes_range.size()); + BOOST_ASSERT(v_index < nodes_range.size()); + + const extractor::QueryNode &u = internal_to_external_node_map[nodes_range[u_index]]; + const extractor::QueryNode &v = internal_to_external_node_map[nodes_range[v_index]]; + + const double segment_length = util::coordinate_calculation::greatCircleDistance( + util::Coordinate{u.lon, u.lat}, util::Coordinate{v.lon, v.lat}); + + auto fwd_source = LUA_SOURCE, rev_source = LUA_SOURCE; + if (auto value = segment_speed_lookup({u.node_id, v.node_id})) + { + EdgeWeight new_segment_weight, new_segment_duration; + getNewWeight(config, + weight_multiplier, + *value, + segment_length, + segment_data.ForwardWeight(geometry_id, segment_offset), + u.node_id, + v.node_id, + new_segment_weight, + new_segment_duration); + + segment_data.ForwardWeight(geometry_id, segment_offset) = new_segment_weight; + segment_data.ForwardDuration(geometry_id, segment_offset) = new_segment_duration; + segment_data.ForwardDatasource(geometry_id, segment_offset) = value->source; + fwd_source = value->source; + } + + if (auto value = segment_speed_lookup({v.node_id, u.node_id})) + { + EdgeWeight new_segment_weight, new_segment_duration; + getNewWeight(config, + weight_multiplier, + *value, + segment_length, + segment_data.ReverseWeight(geometry_id, segment_offset), + v.node_id, + u.node_id, + new_segment_weight, + new_segment_duration); + + segment_data.ReverseWeight(geometry_id, segment_offset) = new_segment_weight; + segment_data.ReverseDuration(geometry_id, segment_offset) = new_segment_duration; + segment_data.ReverseDatasource(geometry_id, segment_offset) = value->source; + rev_source = value->source; + } + + // count statistics for logging + counters[fwd_source] += 1; + counters[rev_source] += 1; + } + }); // parallel_for_each + + counters_type merged_counters(num_counters, 0); + for (const auto &counters : segment_speeds_counters) + { + for (std::size_t i = 0; i < counters.size(); i++) + { + merged_counters[i] += counters[i]; + } + } + + for (std::size_t i = 0; i < merged_counters.size(); i++) + { + if (i == LUA_SOURCE) + { + util::Log() << "Used " << merged_counters[LUA_SOURCE] + << " speeds from LUA profile or input map"; + } + else + { + // segments_speeds_counters has 0 as LUA, segment_speed_filenames not, thus we need + // to susbstract 1 to avoid off-by-one error + util::Log() << "Used " << merged_counters[i] << " speeds from " + << config.segment_speed_lookup_paths[i - 1]; + } + } + + const auto save_geometries = [&]() { + // Now save out the updated compressed geometries + extractor::io::write(config.geometry_path, segment_data); + }; + + const auto save_datastore_names = [&]() { + extractor::Datasources sources; + DatasourceID source = 0; + sources.SetSourceName(source++, "lua profile"); + + // Only write the filename, without path or extension. + // This prevents information leakage, and keeps names short + // for rendering in the debug tiles. + for (auto const &name : config.segment_speed_lookup_paths) + { + sources.SetSourceName(source++, boost::filesystem::path(name).stem().string()); + } + + extractor::io::write(config.datasource_names_path, sources); + }; + + tbb::parallel_invoke(save_geometries, save_datastore_names); +} +} + Updater::NumNodesAndEdges Updater::LoadAndUpdateEdgeExpandedGraph() const { std::vector edge_based_edge_list; @@ -187,26 +366,8 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e util::Log() << "Opening " << config.edge_based_graph_path; - auto mmap_file = [](const std::string &filename, boost::interprocess::mode_t mode) { - using boost::interprocess::file_mapping; - using boost::interprocess::mapped_region; - - try - { - const file_mapping mapping{filename.c_str(), mode}; - mapped_region region{mapping, mode}; - region.advise(mapped_region::advice_sequential); - return region; - } - catch (const std::exception &e) - { - util::Log(logERROR) << "Error while trying to mmap " + filename + ": " + e.what(); - throw; - } - }; - const auto edge_based_graph_region = - mmap_file(config.edge_based_graph_path, boost::interprocess::read_only); + mmapFile(config.edge_based_graph_path, boost::interprocess::read_only); const bool update_edge_weights = !config.segment_speed_lookup_paths.empty(); const bool update_turn_penalties = !config.turn_penalty_lookup_paths.empty(); @@ -214,7 +375,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e const auto turn_penalties_index_region = [&] { if (update_edge_weights || update_turn_penalties) { - return mmap_file(config.turn_penalties_index_path, boost::interprocess::read_only); + return mmapFile(config.turn_penalties_index_path, boost::interprocess::read_only); } return boost::interprocess::mapped_region(); }(); @@ -222,7 +383,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e const auto edge_segment_region = [&] { if (update_edge_weights || update_turn_penalties) { - return mmap_file(config.edge_segment_lookup_path, boost::interprocess::read_only); + return mmapFile(config.edge_segment_lookup_path, boost::interprocess::read_only); } return boost::interprocess::mapped_region(); }(); @@ -270,178 +431,11 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e auto segment_speed_lookup = csv::readSegmentValues(config.segment_speed_lookup_paths); auto turn_penalty_lookup = csv::readTurnValues(config.turn_penalty_lookup_paths); - std::vector internal_to_external_node_map; - extractor::SegmentDataContainer segment_data; - - const auto maybe_load_internal_to_external_node_map = [&] { - if (!update_edge_weights) - return; - - storage::io::FileReader nodes_file(config.node_based_graph_path, - storage::io::FileReader::HasNoFingerprint); - - nodes_file.DeserializeVector(internal_to_external_node_map); - }; - - const auto maybe_load_geometries = [&] { - if (!update_edge_weights) - return; - - extractor::io::read(config.geometry_path, segment_data); - }; - - // Folds all our actions into independently concurrently executing lambdas - tbb::parallel_invoke(maybe_load_internal_to_external_node_map, maybe_load_geometries); - if (update_edge_weights) { - // Here, we have to update the compressed geometry weights - // First, we need the external-to-internal node lookup table - - // Now, we iterate over all the segments stored in the StaticRTree, updating - // the packed geometry weights in the `.geometries` file (note: we do not - // update the RTree itself, we just use the leaf nodes to iterate over all segments) - using LeafNode = util::StaticRTree::LeafNode; - - using boost::interprocess::mapped_region; - - auto region = mmap_file(config.rtree_leaf_path.c_str(), boost::interprocess::read_only); - region.advise(mapped_region::advice_willneed); - BOOST_ASSERT(is_aligned(region.get_address())); - - const auto bytes = region.get_size(); - const auto first = static_cast(region.get_address()); - const auto last = first + (bytes / sizeof(LeafNode)); - - // vector to count used speeds for logging - // size offset by one since index 0 is used for speeds not from external file - using counters_type = std::vector; - std::size_t num_counters = config.segment_speed_lookup_paths.size() + 1; - tbb::enumerable_thread_specific segment_speeds_counters( - counters_type(num_counters, 0)); - const constexpr auto LUA_SOURCE = 0; - - tbb::parallel_for_each(first, last, [&](const LeafNode ¤t_node) { - auto &counters = segment_speeds_counters.local(); - for (size_t i = 0; i < current_node.object_count; i++) - { - const auto &leaf_object = current_node.objects[i]; - - const auto geometry_id = leaf_object.packed_geometry_id; - auto nodes_range = segment_data.GetForwardGeometry(geometry_id); - - const auto segment_offset = leaf_object.fwd_segment_position; - const auto u_index = segment_offset; - const auto v_index = segment_offset + 1; - - BOOST_ASSERT(u_index < nodes_range.size()); - BOOST_ASSERT(v_index < nodes_range.size()); - - const extractor::QueryNode &u = internal_to_external_node_map[nodes_range[u_index]]; - const extractor::QueryNode &v = internal_to_external_node_map[nodes_range[v_index]]; - - const double segment_length = util::coordinate_calculation::greatCircleDistance( - util::Coordinate{u.lon, u.lat}, util::Coordinate{v.lon, v.lat}); - - auto fwd_source = LUA_SOURCE, rev_source = LUA_SOURCE; - if (auto value = segment_speed_lookup({u.node_id, v.node_id})) - { - EdgeWeight new_segment_weight, new_segment_duration; - GetNewWeight(config, - weight_multiplier, - *value, - segment_length, - segment_data.ForwardWeight(geometry_id, segment_offset), - u.node_id, - v.node_id, - new_segment_weight, - new_segment_duration); - - segment_data.ForwardWeight(geometry_id, segment_offset) = new_segment_weight; - segment_data.ForwardDuration(geometry_id, segment_offset) = - new_segment_duration; - segment_data.ForwardDatasource(geometry_id, segment_offset) = value->source; - fwd_source = value->source; - } - - if (auto value = segment_speed_lookup({v.node_id, u.node_id})) - { - EdgeWeight new_segment_weight, new_segment_duration; - GetNewWeight(config, - weight_multiplier, - *value, - segment_length, - segment_data.ReverseWeight(geometry_id, segment_offset), - v.node_id, - u.node_id, - new_segment_weight, - new_segment_duration); - - segment_data.ReverseWeight(geometry_id, segment_offset) = new_segment_weight; - segment_data.ReverseDuration(geometry_id, segment_offset) = - new_segment_duration; - segment_data.ReverseDatasource(geometry_id, segment_offset) = value->source; - rev_source = value->source; - } - - // count statistics for logging - counters[fwd_source] += 1; - counters[rev_source] += 1; - } - }); // parallel_for_each - - counters_type merged_counters(num_counters, 0); - for (const auto &counters : segment_speeds_counters) - { - for (std::size_t i = 0; i < counters.size(); i++) - { - merged_counters[i] += counters[i]; - } - } - - for (std::size_t i = 0; i < merged_counters.size(); i++) - { - if (i == LUA_SOURCE) - { - util::Log() << "Used " << merged_counters[LUA_SOURCE] - << " speeds from LUA profile or input map"; - } - else - { - // segments_speeds_counters has 0 as LUA, segment_speed_filenames not, thus we need - // to susbstract 1 to avoid off-by-one error - util::Log() << "Used " << merged_counters[i] << " speeds from " - << config.segment_speed_lookup_paths[i - 1]; - } - } + updaterSegmentData(config, profile_properties, segment_speed_lookup); } - const auto maybe_save_geometries = [&] { - if (!update_edge_weights) - return; - - // Now save out the updated compressed geometries - extractor::io::write(config.geometry_path, segment_data); - }; - - const auto save_datastore_names = [&] { - extractor::Datasources sources; - DatasourceID source = 0; - sources.SetSourceName(source++, "lua profile"); - - // Only write the filename, without path or extension. - // This prevents information leakage, and keeps names short - // for rendering in the debug tiles. - for (auto const &name : config.segment_speed_lookup_paths) - { - sources.SetSourceName(source++, boost::filesystem::path(name).stem().string()); - } - - extractor::io::write(config.datasource_names_path, sources); - }; - - tbb::parallel_invoke(maybe_save_geometries, save_datastore_names); - std::vector turn_weight_penalties; std::vector turn_duration_penalties; @@ -517,10 +511,10 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e if (value->speed == 0) return true; - segment_duration = ConvertToDuration(segment.segment_length, value->speed); + segment_duration = convertToDuration(segment.segment_length, value->speed); segment_weight = - ConvertToWeight(value->weight, weight_multiplier, segment_duration); + convertToWeight(value->weight, weight_multiplier, segment_duration); } // Update the edge weight and the next OSM node ID @@ -603,7 +597,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e if (config.turn_penalty_lookup_paths.empty()) { // don't check weights consistency with turn updates that can break assertion // condition with turn weight penalties negative updates - CheckWeightsConsistency(config, edge_based_edge_list); + checkWeightsConsistency(config, edge_based_edge_list); } #endif