From 990e411361267df903a3726ebe3af2183712b0ef Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 19 Mar 2018 15:57:57 +0000 Subject: [PATCH] Port turn duration penalties to tar --- include/extractor/files.hpp | 67 +++++++++++++++++--- src/extractor/edge_based_graph_factory.cpp | 13 +--- src/storage/storage.cpp | 58 ++++++++--------- src/updater/updater.cpp | 72 +++++++++------------- 4 files changed, 111 insertions(+), 99 deletions(-) diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index bcb9cf353..9954ba68a 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -269,8 +269,9 @@ inline void writeTurnLaneDescriptions(const boost::filesystem::path &path, template inline void readTurnLaneData(const boost::filesystem::path &path, TurnLaneDataT &turn_lane_data) { - static_assert(std::is_same::value, - ""); + static_assert( + std::is_same::value, + ""); const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; @@ -281,10 +282,11 @@ inline void readTurnLaneData(const boost::filesystem::path &path, TurnLaneDataT // writes .osrm.tld template inline void writeTurnLaneData(const boost::filesystem::path &path, - const TurnLaneDataT &turn_lane_data) + const TurnLaneDataT &turn_lane_data) { - static_assert(std::is_same::value, - ""); + static_assert( + std::is_same::value, + ""); const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; storage::tar::FileWriter writer{path, fingerprint}; @@ -292,7 +294,6 @@ inline void writeTurnLaneData(const boost::filesystem::path &path, storage::serialization::write(writer, "/common/turn_lanes/data", turn_lane_data); } - // reads .osrm.maneuver_overrides template inline void readManeuverOverrides(const boost::filesystem::path &path, @@ -302,8 +303,10 @@ inline void readManeuverOverrides(const boost::filesystem::path &path, const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; - storage::serialization::read(reader, "/common/maneuver_overrides/overrides", maneuver_overrides); - storage::serialization::read(reader, "/common/maneuver_overrides/node_sequences", node_sequences); + storage::serialization::read( + reader, "/common/maneuver_overrides/overrides", maneuver_overrides); + storage::serialization::read( + reader, "/common/maneuver_overrides/node_sequences", node_sequences); } // writes .osrm.maneuver_overrides @@ -314,8 +317,52 @@ inline void writeManeuverOverrides(const boost::filesystem::path &path, const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; storage::tar::FileWriter writer{path, fingerprint}; - storage::serialization::write(writer, "/common/maneuver_overrides/overrides", maneuver_overrides); - storage::serialization::write(writer, "/common/maneuver_overrides/node_sequences", node_sequences); + storage::serialization::write( + writer, "/common/maneuver_overrides/overrides", maneuver_overrides); + storage::serialization::write( + writer, "/common/maneuver_overrides/node_sequences", node_sequences); +} + +// writes .osrm.turn_weight_penalties +template +inline void writeTurnWeightPenalty(const boost::filesystem::path &path, + const TurnPenaltyT &turn_penalty) +{ + const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; + storage::tar::FileWriter writer{path, fingerprint}; + + storage::serialization::write(writer, "/common/turn_penalty/weight", turn_penalty); +} + +// read .osrm.turn_weight_penalties +template +inline void readTurnWeightPenalty(const boost::filesystem::path &path, TurnPenaltyT &turn_penalty) +{ + const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; + storage::tar::FileReader reader{path, fingerprint}; + + storage::serialization::read(reader, "/common/turn_penalty/weight", turn_penalty); +} + +// writes .osrm.turn_duration_penalties +template +inline void writeTurnDurationPenalty(const boost::filesystem::path &path, + const TurnPenaltyT &turn_penalty) +{ + const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; + storage::tar::FileWriter writer{path, fingerprint}; + + storage::serialization::write(writer, "/common/turn_penalty/duration", turn_penalty); +} + +// read .osrm.turn_weight_penalties +template +inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPenaltyT &turn_penalty) +{ + const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; + storage::tar::FileReader reader{path, fingerprint}; + + storage::serialization::read(reader, "/common/turn_penalty/duration", turn_penalty); } } } diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index 10831a8b6..c9e5d031c 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -1138,17 +1138,8 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( // write weight penalties per turn BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size()); - { - storage::io::FileWriter writer(turn_weight_penalties_filename, - storage::io::FileWriter::GenerateFingerprint); - storage::serialization::write(writer, turn_weight_penalties); - } - - { - storage::io::FileWriter writer(turn_duration_penalties_filename, - storage::io::FileWriter::GenerateFingerprint); - storage::serialization::write(writer, turn_duration_penalties); - } + files::writeTurnWeightPenalty(turn_weight_penalties_filename, turn_weight_penalties); + files::writeTurnDurationPenalty(turn_duration_penalties_filename, turn_duration_penalties); util::Log() << "Generated " << m_edge_based_node_segments.size() << " edge based node segments"; util::Log() << "Node-based graph contains " << node_based_edge_counter << " edges"; diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index c7b28f5bf..bc80e6d51 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -250,7 +250,6 @@ void Storage::PopulateLayout(DataLayout &layout) layout.SetBlock(DataLayout::NAME_CHAR_DATA, make_block(name_file.GetSize())); } - // Loading information for original edges { io::FileReader edges_file(config.GetPath(".osrm.edges"), io::FileReader::VerifyFingerprint); @@ -289,24 +288,6 @@ void Storage::PopulateLayout(DataLayout &layout) layout.SetBlock(DataLayout::TIMESTAMP, make_block(timestamp_size)); } - // load turn weight penalties - { - io::FileReader turn_weight_penalties_file(config.GetPath(".osrm.turn_weight_penalties"), - io::FileReader::VerifyFingerprint); - const auto number_of_penalties = turn_weight_penalties_file.ReadElementCount64(); - layout.SetBlock(DataLayout::TURN_WEIGHT_PENALTIES, - make_block(number_of_penalties)); - } - - // load turn duration penalties - { - io::FileReader turn_duration_penalties_file(config.GetPath(".osrm.turn_duration_penalties"), - io::FileReader::VerifyFingerprint); - const auto number_of_penalties = turn_duration_penalties_file.ReadElementCount64(); - layout.SetBlock(DataLayout::TURN_DURATION_PENALTIES, - make_block(number_of_penalties)); - } - std::unordered_map name_to_block_id = { {"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST}, {"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST}, @@ -349,8 +330,10 @@ void Storage::PopulateLayout(DataLayout &layout) {"/ch/edge_filter/7", DataLayout::CH_EDGE_FILTER_7}, {"/common/intersection_bearings/bearing_values", DataLayout::BEARING_VALUES}, {"/common/intersection_bearings/node_to_class_id", DataLayout::BEARING_CLASSID}, - {"/common/intersection_bearings/class_id_to_ranges/block_offsets", DataLayout::BEARING_OFFSETS}, - {"/common/intersection_bearings/class_id_to_ranges/diff_blocks", DataLayout::BEARING_BLOCKS}, + {"/common/intersection_bearings/class_id_to_ranges/block_offsets", + DataLayout::BEARING_OFFSETS}, + {"/common/intersection_bearings/class_id_to_ranges/diff_blocks", + DataLayout::BEARING_BLOCKS}, {"/common/entry_classes", DataLayout::ENTRY_CLASS}, {"/common/properties", DataLayout::PROPERTIES}, {"/common/coordinates", DataLayout::COORDINATE_LIST}, @@ -371,6 +354,8 @@ void Storage::PopulateLayout(DataLayout &layout) {"/common/turn_lanes/data", DataLayout::TURN_LANE_DATA}, {"/common/maneuver_overrides/overrides", DataLayout::MANEUVER_OVERRIDES}, {"/common/maneuver_overrides/node_sequences", DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES}, + {"/common/turn_penalty/weight", DataLayout::TURN_WEIGHT_PENALTIES}, + {"/common/turn_penalty/duration", DataLayout::TURN_DURATION_PENALTIES}, }; std::vector blocks; @@ -391,6 +376,8 @@ void Storage::PopulateLayout(DataLayout &layout) {REQUIRED, config.GetPath(".osrm.tls")}, {REQUIRED, config.GetPath(".osrm.tld")}, {REQUIRED, config.GetPath(".osrm.maneuver_overrides")}, + {REQUIRED, config.GetPath(".osrm.turn_weight_penalties")}, + {REQUIRED, config.GetPath(".osrm.turn_duration_penalties")}, }; for (const auto &file : tar_files) @@ -403,7 +390,8 @@ void Storage::PopulateLayout(DataLayout &layout) { if (std::get<0>(file) == REQUIRED) { - throw util::exception("Could not find required filed: " + std::get<1>(file).string()); + throw util::exception("Could not find required filed: " + + std::get<1>(file).string()); } } } @@ -640,22 +628,24 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // load turn weight penalties { - io::FileReader turn_weight_penalties_file(config.GetPath(".osrm.turn_weight_penalties"), - io::FileReader::VerifyFingerprint); - const auto number_of_penalties = turn_weight_penalties_file.ReadElementCount64(); - const auto turn_weight_penalties_ptr = - layout.GetBlockPtr(memory_ptr, DataLayout::TURN_WEIGHT_PENALTIES); - turn_weight_penalties_file.ReadInto(turn_weight_penalties_ptr, number_of_penalties); + auto turn_duration_penalties_ptr = layout.GetBlockPtr( + memory_ptr, storage::DataLayout::TURN_WEIGHT_PENALTIES); + util::vector_view turn_duration_penalties( + turn_duration_penalties_ptr, + layout.GetBlockEntries(storage::DataLayout::TURN_WEIGHT_PENALTIES)); + extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"), + turn_duration_penalties); } // load turn duration penalties { - io::FileReader turn_duration_penalties_file(config.GetPath(".osrm.turn_duration_penalties"), - io::FileReader::VerifyFingerprint); - const auto number_of_penalties = turn_duration_penalties_file.ReadElementCount64(); - const auto turn_duration_penalties_ptr = - layout.GetBlockPtr(memory_ptr, DataLayout::TURN_DURATION_PENALTIES); - turn_duration_penalties_file.ReadInto(turn_duration_penalties_ptr, number_of_penalties); + auto turn_duration_penalties_ptr = layout.GetBlockPtr( + memory_ptr, storage::DataLayout::TURN_DURATION_PENALTIES); + util::vector_view turn_duration_penalties( + turn_duration_penalties_ptr, + layout.GetBlockEntries(storage::DataLayout::TURN_DURATION_PENALTIES)); + extractor::files::readTurnDurationPenalty(config.GetPath(".osrm.turn_duration_penalties"), + turn_duration_penalties); } // store timestamp diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index fd337f4ae..77068ee00 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -162,8 +162,10 @@ updateSegmentData(const UpdaterConfig &config, // closure to convert SpeedSource value to weight and count fallbacks to durations std::atomic fallbacks_to_duration{0}; - auto convertToWeight = [&profile_properties, &fallbacks_to_duration]( - const SegmentWeight &existing_weight, const SpeedSource &value, double distance_in_meters) { + auto convertToWeight = [&profile_properties, + &fallbacks_to_duration](const SegmentWeight &existing_weight, + const SpeedSource &value, + double distance_in_meters) { double rate = std::numeric_limits::quiet_NaN(); @@ -571,37 +573,24 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e std::vector turn_duration_penalties; if (update_edge_weights || update_turn_penalties || update_conditional_turns) { - const auto load_segment_data = [&] { - extractor::files::readSegmentData(config.GetPath(".osrm.geometry"), segment_data); - }; + tbb::parallel_invoke( + [&] { + extractor::files::readSegmentData(config.GetPath(".osrm.geometry"), segment_data); + }, + [&] { extractor::files::readNodeData(config.GetPath(".osrm.ebg_nodes"), node_data); }, - const auto load_node_data = [&] { - extractor::files::readNodeData(config.GetPath(".osrm.ebg_nodes"), node_data); - }; - - const auto load_turn_weight_penalties = [&] { - using storage::io::FileReader; - FileReader reader(config.GetPath(".osrm.turn_weight_penalties"), - FileReader::VerifyFingerprint); - storage::serialization::read(reader, turn_weight_penalties); - }; - - const auto load_turn_duration_penalties = [&] { - using storage::io::FileReader; - FileReader reader(config.GetPath(".osrm.turn_duration_penalties"), - FileReader::VerifyFingerprint); - storage::serialization::read(reader, turn_duration_penalties); - }; - - const auto load_profile_properties = [&] { - extractor::files::readProfileProperties(config.GetPath(".osrm.properties"), profile_properties); - }; - - tbb::parallel_invoke(load_node_data, - load_segment_data, - load_turn_weight_penalties, - load_turn_duration_penalties, - load_profile_properties); + [&] { + extractor::files::readTurnWeightPenalty( + config.GetPath(".osrm.turn_weight_penalties"), turn_weight_penalties); + }, + [&] { + extractor::files::readTurnDurationPenalty( + config.GetPath(".osrm.turn_duration_penalties"), turn_duration_penalties); + }, + [&] { + extractor::files::readProfileProperties(config.GetPath(".osrm.properties"), + profile_properties); + }); } std::vector conditional_turns; @@ -777,9 +766,9 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e { if (turn_weight_penalty < 0) { - util::Log(logWARNING) << "turn penalty " << turn_weight_penalty - << " is too negative: clamping turn weight to " - << weight_min_value; + util::Log(logWARNING) + << "turn penalty " << turn_weight_penalty + << " is too negative: clamping turn weight to " << weight_min_value; turn_weight_penalty = weight_min_value - new_weight; turn_weight_penalties[edge.data.turn_id] = turn_weight_penalty; } @@ -808,19 +797,14 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e if (update_turn_penalties || update_conditional_turns) { - const auto save_penalties = [](const auto &filename, const auto &data) -> void { - storage::io::FileWriter writer(filename, storage::io::FileWriter::GenerateFingerprint); - storage::serialization::write(writer, data); - }; - tbb::parallel_invoke( [&] { - save_penalties(config.GetPath(".osrm.turn_weight_penalties"), - turn_weight_penalties); + extractor::files::writeTurnWeightPenalty( + config.GetPath(".osrm.turn_weight_penalties"), turn_weight_penalties); }, [&] { - save_penalties(config.GetPath(".osrm.turn_duration_penalties"), - turn_duration_penalties); + extractor::files::writeTurnDurationPenalty( + config.GetPath(".osrm.turn_duration_penalties"), turn_duration_penalties); }); }