diff --git a/CHANGELOG.md b/CHANGELOG.md index c00c30248..130c42153 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Unreleased - Changes from 5.23.0 + - Misc: + - CHANGED: Unify `.osrm.turn_penalites_index` dump processing same with `.osrm.turn_weight_penalties` and `.osrm.turn_duration_penalties` [#5868](https://github.com/Project-OSRM/osrm-backend/pull/5868) - Infrastructure - CHANGED: Bundled protozero updated to v1.7.0. [#5858](https://github.com/Project-OSRM/osrm-backend/pull/5858) - Windows: diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 493c54765..a463bbbc5 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -399,6 +399,28 @@ inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPen storage::serialization::read(reader, "/common/turn_penalty/duration", turn_penalty); } +// writes .osrm.turn_penalties_index +template +inline void writeTurnPenaltiesIndex(const boost::filesystem::path &path, + const TurnIndexT &turn_penalties_index) +{ + const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; + storage::tar::FileWriter writer{path, fingerprint}; + + storage::serialization::write(writer, "/extractor/turn_index", turn_penalties_index); +} + +// read .osrm.turn_penalties_index +template +inline void readTurnPenaltiesIndex(const boost::filesystem::path &path, + TurnIndexT &turn_penalties_index) +{ + const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; + storage::tar::FileReader reader{path, fingerprint}; + + storage::serialization::read(reader, "/extractor/turn_index", turn_penalties_index); +} + // writes .osrm.restrictions template inline void writeConditionalRestrictions(const boost::filesystem::path &path, diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index 12e4ce0ec..3684c1478 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -53,9 +53,6 @@ template <> struct hash> }; } -// Buffer size of turn_indexes_write_buffer to reduce number of write(v) syscals -const constexpr int TURN_INDEX_WRITE_BUFFER_SIZE = 1000; - namespace osrm { namespace extractor @@ -449,10 +446,6 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( std::size_t node_based_edge_counter = 0; - storage::tar::FileWriter turn_penalties_index_file( - turn_penalties_index_filename, storage::tar::FileWriter::GenerateFingerprint); - turn_penalties_index_file.WriteFrom("/extractor/turn_index", (char *)nullptr, 0); - SuffixTable street_name_suffix_table(scripting_environment); const auto &turn_lanes_data = transformTurnLaneMapIntoArrays(lane_description_map); intersection::MergableRoadDetector mergable_road_detector(m_node_based_graph, @@ -468,6 +461,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( // FIXME these need to be tuned in pre-allocated size std::vector turn_weight_penalties; std::vector turn_duration_penalties; + std::vector turn_penalties_index; // Now, renumber all our maneuver overrides to use edge-based-nodes std::vector storage_maneuver_overrides; @@ -487,14 +481,6 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( { const NodeID node_count = m_node_based_graph.GetNumberOfNodes(); - // Because we write TurnIndexBlock data as we go, we'll - // buffer them into groups of 1000 to reduce the syscall - // count by 1000x. This doesn't need much memory, but - // greatly reduces the syscall overhead of writing lots - // of small objects - std::vector turn_indexes_write_buffer; - turn_indexes_write_buffer.reserve(TURN_INDEX_WRITE_BUFFER_SIZE); - // This struct is the buffered output of the `processor_stage`. This data is // appended to the various output arrays/files by the `output_stage`. // same as IntersectionData, but grouped with edge to allow sorting after creating. @@ -510,7 +496,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( m_edge_based_edge_list.push_back(edge_with_data.edge); turn_weight_penalties.push_back(edge_with_data.turn_weight_penalty); turn_duration_penalties.push_back(edge_with_data.turn_duration_penalty); - turn_indexes_write_buffer.push_back(edge_with_data.turn_index); + turn_penalties_index.push_back(edge_with_data.turn_index); }; struct EdgesPipelineBuffer @@ -1054,15 +1040,6 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( // NOTE: potential overflow here if we hit 2^32 routable edges BOOST_ASSERT(m_edge_based_edge_list.size() <= std::numeric_limits::max()); - // Buffer writes to reduce syscall count - if (turn_indexes_write_buffer.size() >= TURN_INDEX_WRITE_BUFFER_SIZE) - { - turn_penalties_index_file.ContinueFrom("/extractor/turn_index", - turn_indexes_write_buffer.data(), - turn_indexes_write_buffer.size()); - turn_indexes_write_buffer.clear(); - } - // Copy via-way restrictions delayed data delayed_data.insert( delayed_data.end(), buffer->delayed_data.begin(), buffer->delayed_data.end()); @@ -1118,15 +1095,6 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( storage_maneuver_overrides.push_back(storage_override); } - - // Flush the turn_indexes_write_buffer if it's not empty - if (!turn_indexes_write_buffer.empty()) - { - turn_penalties_index_file.ContinueFrom("/extractor/turn_index", - turn_indexes_write_buffer.data(), - turn_indexes_write_buffer.size()); - turn_indexes_write_buffer.clear(); - } } { util::Log() << "Sorting and writing " << storage_maneuver_overrides.size() @@ -1162,9 +1130,11 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( indexed_conditionals); // write weight penalties per turn - BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size()); + BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size() && + turn_weight_penalties.size() == turn_penalties_index.size()); files::writeTurnWeightPenalty(turn_weight_penalties_filename, turn_weight_penalties); files::writeTurnDurationPenalty(turn_duration_penalties_filename, turn_duration_penalties); + files::writeTurnPenaltiesIndex(turn_penalties_index_filename, turn_penalties_index); util::Log() << "Generated " << m_edge_based_node_segments.size() << " edge based node segments"; util::Log() << "Node-based graph contains " << node_based_edge_counter << " edges"; diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index f4dc902a3..552910842 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -432,6 +432,10 @@ updateTurnPenalties(const UpdaterConfig &config, { const auto weight_multiplier = profile_properties.GetWeightMultiplier(); + // [NOTE] turn_index_blocks could be simply loaded by `files::readTurnPenaltiesIndex()`, + // however, we leave the below mmap to keep compatiblity. + // Use `files::readTurnPenaltiesIndex()` instead once the compatiblity is not that + // important. // Mapped file pointer for turn indices boost::iostreams::mapped_file_source turn_index_region; const extractor::lookup::TurnIndexBlock *turn_index_blocks;