From 1e2c40337c31546c1d45bec161dcbb8ffa82a5b5 Mon Sep 17 00:00:00 2001 From: Kajari Ghosh Date: Wed, 26 Sep 2018 01:25:06 -0400 Subject: [PATCH] write distances out in a seperate file for mld --- .../extractor/edge_based_graph_factory.hpp | 2 + include/extractor/extractor.hpp | 1 + include/extractor/files.hpp | 28 ++ include/updater/updater.hpp | 6 + src/customize/customizer.cpp | 5 +- src/extractor/edge_based_graph_factory.cpp | 12 + src/extractor/extractor.cpp | 10 +- src/partitioner/partitioner.cpp | 10 +- src/updater/updater.cpp | 294 ++++++++++++++++++ 9 files changed, 360 insertions(+), 8 deletions(-) diff --git a/include/extractor/edge_based_graph_factory.hpp b/include/extractor/edge_based_graph_factory.hpp index d8e8aeedc..388e0d75e 100644 --- a/include/extractor/edge_based_graph_factory.hpp +++ b/include/extractor/edge_based_graph_factory.hpp @@ -92,6 +92,7 @@ class EdgeBasedGraphFactory void GetStartPointMarkers(std::vector &node_is_startpoint); void GetEdgeBasedNodeWeights(std::vector &output_node_weights); void GetEdgeBasedNodeDurations(std::vector &output_node_durations); + void GetEdgeBasedNodeDistances(std::vector &output_node_distances); std::uint32_t GetConnectivityChecksum() const; std::uint64_t GetNumberOfEdgeBasedNodes() const; @@ -119,6 +120,7 @@ class EdgeBasedGraphFactory //! edge-based node std::vector m_edge_based_node_weights; std::vector m_edge_based_node_durations; + std::vector m_edge_based_node_distances; //! list of edge based nodes (compressed segments) std::vector m_edge_based_node_segments; diff --git a/include/extractor/extractor.hpp b/include/extractor/extractor.hpp index 31b157658..db571475b 100644 --- a/include/extractor/extractor.hpp +++ b/include/extractor/extractor.hpp @@ -88,6 +88,7 @@ class Extractor std::vector &node_is_startpoint, std::vector &edge_based_node_weights, std::vector &edge_based_node_durations, + std::vector &edge_based_node_distances, util::DeallocatingVector &edge_based_edge_list, std::uint32_t &connectivity_checksum); diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 91f98825a..437d2f637 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -462,6 +462,34 @@ void readEdgeBasedNodeWeights(const boost::filesystem::path &path, NodeWeigtsVec storage::serialization::read(reader, "/extractor/edge_based_node_weights", weights); } +template +void readEdgeBasedNodeWeightsDurationsDistances(const boost::filesystem::path &path, + NodeWeigtsVectorT &weights, + NodeDurationsVectorT &durations, + NodeDistancesVectorT &distances) +{ + const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; + storage::tar::FileReader reader{path, fingerprint}; + + storage::serialization::read(reader, "/extractor/edge_based_node_weights", weights); + storage::serialization::read(reader, "/extractor/edge_based_node_durations", durations); + storage::serialization::read(reader, "/extractor/edge_based_node_distances", distances); +} + +template +void writeEdgeBasedNodeWeightsDurationsDistances(const boost::filesystem::path &path, + const NodeWeigtsVectorT &weights, + const NodeDurationsVectorT &durations, + const NodeDistancesVectorT &distances) +{ + const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; + storage::tar::FileWriter writer{path, fingerprint}; + + storage::serialization::write(writer, "/extractor/edge_based_node_weights", weights); + storage::serialization::write(writer, "/extractor/edge_based_node_durations", durations); + storage::serialization::write(writer, "/extractor/edge_based_node_distances", distances); +} + template void readEdgeBasedNodeWeightsDurations(const boost::filesystem::path &path, NodeWeigtsVectorT &weights, diff --git a/include/updater/updater.hpp b/include/updater/updater.hpp index dcaf5f638..894159a26 100644 --- a/include/updater/updater.hpp +++ b/include/updater/updater.hpp @@ -27,6 +27,12 @@ class Updater std::vector &node_weights, std::vector &node_durations, // TODO: to be deleted std::uint32_t &connectivity_checksum) const; + EdgeID + LoadAndUpdateEdgeExpandedGraph(std::vector &edge_based_edge_list, + std::vector &node_weights, + std::vector &node_durations, // TODO: to be deleted + std::vector &node_distances, // TODO: to be deleted + std::uint32_t &connectivity_checksum) const; private: UpdaterConfig config; diff --git a/src/customize/customizer.cpp b/src/customize/customizer.cpp index 03139c54b..eaba4ebd8 100644 --- a/src/customize/customizer.cpp +++ b/src/customize/customizer.cpp @@ -76,13 +76,14 @@ auto LoadAndUpdateEdgeExpandedGraph(const CustomizationConfig &config, const partitioner::MultiLevelPartition &mlp, std::vector &node_weights, std::vector &node_durations, + std::vector &node_distances, std::uint32_t &connectivity_checksum) { updater::Updater updater(config.updater_config); std::vector edge_based_edge_list; EdgeID num_nodes = updater.LoadAndUpdateEdgeExpandedGraph( - edge_based_edge_list, node_weights, node_durations, connectivity_checksum); + edge_based_edge_list, node_weights, node_durations, node_distances, connectivity_checksum); auto directed = partitioner::splitBidirectionalEdges(edge_based_edge_list); @@ -129,7 +130,7 @@ int Customizer::Run(const CustomizationConfig &config) node_distances; // TODO: is this and the above still to be removed later? dont think so std::uint32_t connectivity_checksum = 0; auto graph = LoadAndUpdateEdgeExpandedGraph( - config, mlp, node_weights, node_durations, connectivity_checksum); + config, mlp, node_weights, node_durations, node_distances, connectivity_checksum); BOOST_ASSERT(graph.GetNumberOfNodes() == node_weights.size()); std::for_each(node_weights.begin(), node_weights.end(), [](auto &w) { w &= 0x7fffffff; }); util::Log() << "Loaded edge based graph: " << graph.GetNumberOfEdges() << " edges, " diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index ab9bbf67b..3119c1df2 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -114,6 +114,13 @@ void EdgeBasedGraphFactory::GetEdgeBasedNodeDurations( swap(m_edge_based_node_durations, output_node_durations); } +void EdgeBasedGraphFactory::GetEdgeBasedNodeDistances( + std::vector &output_node_distances) +{ + using std::swap; // Koenig swap + swap(m_edge_based_node_distances, output_node_distances); +} + std::uint32_t EdgeBasedGraphFactory::GetConnectivityChecksum() const { return m_connectivity_checksum; @@ -293,6 +300,7 @@ unsigned EdgeBasedGraphFactory::LabelEdgeBasedNodes() // (edge-based nodes) m_edge_based_node_weights.reserve(4 * m_node_based_graph.GetNumberOfNodes()); m_edge_based_node_durations.reserve(4 * m_node_based_graph.GetNumberOfNodes()); + m_edge_based_node_distances.reserve(4 * m_node_based_graph.GetNumberOfNodes()); nbe_to_ebn_mapping.resize(m_node_based_graph.GetEdgeCapacity(), SPECIAL_NODEID); // renumber edge based node of outgoing edges @@ -310,6 +318,7 @@ unsigned EdgeBasedGraphFactory::LabelEdgeBasedNodes() m_edge_based_node_weights.push_back(edge_data.weight); m_edge_based_node_durations.push_back(edge_data.duration); + m_edge_based_node_distances.push_back(edge_data.distance); BOOST_ASSERT(numbered_edges_count < m_node_based_graph.GetNumberOfEdges()); nbe_to_ebn_mapping[current_edge] = numbered_edges_count; @@ -407,6 +416,8 @@ EdgeBasedGraphFactory::GenerateEdgeExpandedNodes(const WayRestrictionMap &way_re m_edge_based_node_weights.push_back(ebn_weight); m_edge_based_node_durations.push_back( m_edge_based_node_durations[nbe_to_ebn_mapping[eid]]); + m_edge_based_node_distances.push_back( + m_edge_based_node_distances[nbe_to_ebn_mapping[eid]]); edge_based_node_id++; progress.PrintStatus(progress_counter++); @@ -416,6 +427,7 @@ EdgeBasedGraphFactory::GenerateEdgeExpandedNodes(const WayRestrictionMap &way_re BOOST_ASSERT(m_edge_based_node_segments.size() == m_edge_based_node_is_startpoint.size()); BOOST_ASSERT(m_number_of_edge_based_nodes == m_edge_based_node_weights.size()); BOOST_ASSERT(m_number_of_edge_based_nodes == m_edge_based_node_durations.size()); + BOOST_ASSERT(m_number_of_edge_based_nodes == m_edge_based_node_distances.size()); util::Log() << "Generated " << m_number_of_edge_based_nodes << " nodes (" << way_restriction_map.NumberOfDuplicatedNodes() diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index f62bb1b39..dc92f7011 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -242,6 +242,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) std::vector node_is_startpoint; std::vector edge_based_node_weights; std::vector edge_based_node_durations; + std::vector edge_based_node_distances; std::uint32_t ebg_connectivity_checksum = 0; // Create a node-based graph from the OSRM file @@ -322,6 +323,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) node_is_startpoint, edge_based_node_weights, edge_based_node_durations, + edge_based_node_distances, edge_based_edge_list, ebg_connectivity_checksum); @@ -345,8 +347,10 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) util::Log() << "Saving edge-based node weights to file."; TIMER_START(timer_write_node_weights); - extractor::files::writeEdgeBasedNodeWeightsDurations( - config.GetPath(".osrm.enw"), edge_based_node_weights, edge_based_node_durations); + extractor::files::writeEdgeBasedNodeWeightsDurationsDistances(config.GetPath(".osrm.enw"), + edge_based_node_weights, + edge_based_node_durations, + edge_based_node_distances); TIMER_STOP(timer_write_node_weights); util::Log() << "Done writing. (" << TIMER_SEC(timer_write_node_weights) << ")"; @@ -736,6 +740,7 @@ EdgeID Extractor::BuildEdgeExpandedGraph( std::vector &node_is_startpoint, std::vector &edge_based_node_weights, std::vector &edge_based_node_durations, + std::vector &edge_based_node_distances, util::DeallocatingVector &edge_based_edge_list, std::uint32_t &connectivity_checksum) { @@ -786,6 +791,7 @@ EdgeID Extractor::BuildEdgeExpandedGraph( edge_based_graph_factory.GetStartPointMarkers(node_is_startpoint); edge_based_graph_factory.GetEdgeBasedNodeWeights(edge_based_node_weights); edge_based_graph_factory.GetEdgeBasedNodeDurations(edge_based_node_durations); + edge_based_graph_factory.GetEdgeBasedNodeDistances(edge_based_node_distances); connectivity_checksum = edge_based_graph_factory.GetConnectivityChecksum(); return number_of_edge_based_nodes; diff --git a/src/partitioner/partitioner.cpp b/src/partitioner/partitioner.cpp index c18089386..38daf2cc3 100644 --- a/src/partitioner/partitioner.cpp +++ b/src/partitioner/partitioner.cpp @@ -147,12 +147,14 @@ int Partitioner::Run(const PartitionerConfig &config) { std::vector node_weights; std::vector node_durations; - extractor::files::readEdgeBasedNodeWeightsDurations( - config.GetPath(".osrm.enw"), node_weights, node_durations); + std::vector node_distances; + extractor::files::readEdgeBasedNodeWeightsDurationsDistances( + config.GetPath(".osrm.enw"), node_weights, node_durations, node_distances); util::inplacePermutation(node_weights.begin(), node_weights.end(), permutation); util::inplacePermutation(node_durations.begin(), node_durations.end(), permutation); - extractor::files::writeEdgeBasedNodeWeightsDurations( - config.GetPath(".osrm.enw"), node_weights, node_durations); + util::inplacePermutation(node_distances.begin(), node_distances.end(), permutation); + extractor::files::writeEdgeBasedNodeWeightsDurationsDistances( + config.GetPath(".osrm.enw"), node_weights, node_durations, node_distances); } { const auto &filename = config.GetPath(".osrm.maneuver_overrides"); diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index f4dc902a3..f11e357ae 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -541,6 +541,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e std::vector &node_durations, std::uint32_t &connectivity_checksum) const { + TIMER_START(load_edges); EdgeID number_of_edge_based_nodes = 0; @@ -812,6 +813,299 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e }); } +#if !defined(NDEBUG) + if (config.turn_penalty_lookup_paths.empty()) + { // don't check weights consistency with turn updates that can break assertion + // condition with turn weight penalties negative updates + checkWeightsConsistency(config, edge_based_edge_list); + } +#endif + + saveDatasourcesNames(config); + + TIMER_STOP(load_edges); + util::Log() << "Done reading edges in " << TIMER_MSEC(load_edges) << "ms."; + return number_of_edge_based_nodes; +} + +EdgeID +Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &edge_based_edge_list, + std::vector &node_weights, + std::vector &node_durations, + std::vector &node_distances, + std::uint32_t &connectivity_checksum) const +{ + TIMER_START(load_edges); + + EdgeID number_of_edge_based_nodes = 0; + std::vector coordinates; + extractor::PackedOSMIDs osm_node_ids; + + extractor::files::readEdgeBasedNodeWeightsDurationsDistances( + config.GetPath(".osrm.enw"), node_weights, node_durations, node_distances); + + extractor::files::readEdgeBasedGraph(config.GetPath(".osrm.ebg"), + number_of_edge_based_nodes, + edge_based_edge_list, + connectivity_checksum); + extractor::files::readNodes(config.GetPath(".osrm.nbg_nodes"), coordinates, osm_node_ids); + + const bool update_conditional_turns = + !config.GetPath(".osrm.restrictions").empty() && config.valid_now; + const bool update_edge_weights = !config.segment_speed_lookup_paths.empty(); + const bool update_turn_penalties = !config.turn_penalty_lookup_paths.empty(); + + if (!update_edge_weights && !update_turn_penalties && !update_conditional_turns) + { + saveDatasourcesNames(config); + return number_of_edge_based_nodes; + } + + if (config.segment_speed_lookup_paths.size() + config.turn_penalty_lookup_paths.size() > 255) + throw util::exception("Limit of 255 segment speed and turn penalty files each reached" + + SOURCE_REF); + + extractor::EdgeBasedNodeDataContainer node_data; + extractor::SegmentDataContainer segment_data; + extractor::ProfileProperties profile_properties; + std::vector turn_weight_penalties; + std::vector turn_duration_penalties; + if (update_edge_weights || update_turn_penalties || update_conditional_turns) + { + tbb::parallel_invoke( + [&] { + extractor::files::readSegmentData(config.GetPath(".osrm.geometry"), segment_data); + }, + [&] { extractor::files::readNodeData(config.GetPath(".osrm.ebg_nodes"), node_data); }, + + [&] { + extractor::files::readTurnWeightPenalty( + config.GetPath(".osrm.turn_weight_penalties"), turn_weight_penalties); + }, + [&] { + extractor::files::readTurnDurationPenalty( + config.GetPath(".osrm.turn_duration_penalties"), turn_duration_penalties); + }, + [&] { + extractor::files::readProfileProperties(config.GetPath(".osrm.properties"), + profile_properties); + }); + } + + std::vector conditional_turns; + if (update_conditional_turns) + { + extractor::files::readConditionalRestrictions(config.GetPath(".osrm.restrictions"), + conditional_turns); + } + + tbb::concurrent_vector updated_segments; + if (update_edge_weights) + { + auto segment_speed_lookup = csv::readSegmentValues(config.segment_speed_lookup_paths); + + TIMER_START(segment); + updated_segments = updateSegmentData(config, + profile_properties, + segment_speed_lookup, + segment_data, + coordinates, + osm_node_ids); + // Now save out the updated compressed geometries + extractor::files::writeSegmentData(config.GetPath(".osrm.geometry"), segment_data); + TIMER_STOP(segment); + util::Log() << "Updating segment data took " << TIMER_MSEC(segment) << "ms."; + } + + auto turn_penalty_lookup = csv::readTurnValues(config.turn_penalty_lookup_paths); + if (update_turn_penalties) + { + auto updated_turn_penalties = updateTurnPenalties(config, + profile_properties, + turn_penalty_lookup, + turn_weight_penalties, + turn_duration_penalties, + osm_node_ids); + const auto offset = updated_segments.size(); + updated_segments.resize(offset + updated_turn_penalties.size()); + // we need to re-compute all edges that have updated turn penalties. + // this marks it for re-computation + std::transform(updated_turn_penalties.begin(), + updated_turn_penalties.end(), + updated_segments.begin() + offset, + [&node_data, &edge_based_edge_list](const std::uint64_t turn_id) { + const auto node_id = edge_based_edge_list[turn_id].source; + return node_data.GetGeometryID(node_id); + }); + } + + if (update_conditional_turns) + { + // initialize instance of class that handles time zone resolution + if (config.valid_now <= 0) + { + util::Log(logERROR) << "Given UTC time is invalid: " << config.valid_now; + throw; + } + const Timezoner time_zone_handler = Timezoner(config.tz_file_path, config.valid_now); + + auto updated_turn_penalties = + updateConditionalTurns(turn_weight_penalties, conditional_turns, time_zone_handler); + const auto offset = updated_segments.size(); + updated_segments.resize(offset + updated_turn_penalties.size()); + // we need to re-compute all edges that have updated turn penalties. + // this marks it for re-computation + std::transform(updated_turn_penalties.begin(), + updated_turn_penalties.end(), + updated_segments.begin() + offset, + [&node_data, &edge_based_edge_list](const std::uint64_t turn_id) { + const auto node_id = edge_based_edge_list[turn_id].source; + return node_data.GetGeometryID(node_id); + }); + } + + tbb::parallel_sort(updated_segments.begin(), + updated_segments.end(), + [](const GeometryID lhs, const GeometryID rhs) { + return std::tie(lhs.id, lhs.forward) < std::tie(rhs.id, rhs.forward); + }); + + using WeightAndDuration = std::tuple; + const auto compute_new_weight_and_duration = + [&](const GeometryID geometry_id) -> WeightAndDuration { + EdgeWeight new_weight = 0; + EdgeWeight new_duration = 0; + if (geometry_id.forward) + { + const auto weights = segment_data.GetForwardWeights(geometry_id.id); + for (const auto weight : weights) + { + if (weight == INVALID_SEGMENT_WEIGHT) + { + new_weight = INVALID_EDGE_WEIGHT; + break; + } + new_weight += weight; + } + const auto durations = segment_data.GetForwardDurations(geometry_id.id); + new_duration = std::accumulate(durations.begin(), durations.end(), EdgeWeight{0}); + } + else + { + const auto weights = segment_data.GetReverseWeights(geometry_id.id); + for (const auto weight : weights) + { + if (weight == INVALID_SEGMENT_WEIGHT) + { + new_weight = INVALID_EDGE_WEIGHT; + break; + } + new_weight += weight; + } + const auto durations = segment_data.GetReverseDurations(geometry_id.id); + new_duration = std::accumulate(durations.begin(), durations.end(), EdgeWeight{0}); + } + return std::make_tuple(new_weight, new_duration); + }; + + std::vector accumulated_segment_data(updated_segments.size()); + tbb::parallel_for(tbb::blocked_range(0, updated_segments.size()), + [&](const auto &range) { + for (auto index = range.begin(); index < range.end(); ++index) + { + accumulated_segment_data[index] = + compute_new_weight_and_duration(updated_segments[index]); + } + }); + + const auto update_edge = [&](extractor::EdgeBasedEdge &edge) { + const auto node_id = edge.source; + const auto geometry_id = node_data.GetGeometryID(node_id); + auto updated_iter = std::lower_bound(updated_segments.begin(), + updated_segments.end(), + geometry_id, + [](const GeometryID lhs, const GeometryID rhs) { + return std::tie(lhs.id, lhs.forward) < + std::tie(rhs.id, rhs.forward); + }); + if (updated_iter != updated_segments.end() && updated_iter->id == geometry_id.id && + updated_iter->forward == geometry_id.forward) + { + // Find a segment with zero speed and simultaneously compute the new edge + // weight + EdgeWeight new_weight; + EdgeWeight new_duration; + std::tie(new_weight, new_duration) = + accumulated_segment_data[updated_iter - updated_segments.begin()]; + + // Update the node-weight cache. This is the weight of the edge-based-node + // only, it doesn't include the turn. We may visit the same node multiple times, + // but we should always assign the same value here. + BOOST_ASSERT(edge.source < node_weights.size()); + node_weights[edge.source] = + node_weights[edge.source] & 0x80000000 ? new_weight | 0x80000000 : new_weight; + node_durations[edge.source] = new_duration; + + // We found a zero-speed edge, so we'll skip this whole edge-based-edge + // which + // effectively removes it from the routing network. + if (new_weight == INVALID_EDGE_WEIGHT) + { + edge.data.weight = INVALID_EDGE_WEIGHT; + return; + } + + // Get the turn penalty and update to the new value if required + auto turn_weight_penalty = turn_weight_penalties[edge.data.turn_id]; + auto turn_duration_penalty = turn_duration_penalties[edge.data.turn_id]; + const auto num_nodes = segment_data.GetForwardGeometry(geometry_id.id).size(); + const auto weight_min_value = static_cast(num_nodes); + if (turn_weight_penalty + new_weight < weight_min_value) + { + if (turn_weight_penalty < 0) + { + util::Log(logWARNING) << "turn penalty " << turn_weight_penalty + << " is too negative: clamping turn weight to " + << weight_min_value; + turn_weight_penalty = weight_min_value - new_weight; + turn_weight_penalties[edge.data.turn_id] = turn_weight_penalty; + } + else + { + new_weight = weight_min_value; + } + } + + // Update edge weight + edge.data.weight = new_weight + turn_weight_penalty; + edge.data.duration = new_duration + turn_duration_penalty; + } + }; + + if (updated_segments.size() > 0) + { + tbb::parallel_for(tbb::blocked_range(0, edge_based_edge_list.size()), + [&](const auto &range) { + for (auto index = range.begin(); index < range.end(); ++index) + { + update_edge(edge_based_edge_list[index]); + } + }); + } + + if (update_turn_penalties || update_conditional_turns) + { + tbb::parallel_invoke( + [&] { + extractor::files::writeTurnWeightPenalty( + config.GetPath(".osrm.turn_weight_penalties"), turn_weight_penalties); + }, + [&] { + extractor::files::writeTurnDurationPenalty( + config.GetPath(".osrm.turn_duration_penalties"), turn_duration_penalties); + }); + } + #if !defined(NDEBUG) if (config.turn_penalty_lookup_paths.empty()) { // don't check weights consistency with turn updates that can break assertion