From f1e4349c82df193ded7956615f4d688cff3e2ba0 Mon Sep 17 00:00:00 2001 From: Michael Krasnyk Date: Tue, 25 Apr 2017 11:36:34 +0200 Subject: [PATCH] Move geometry ids, name_ids and travel_modes to EdgeBasedNodeData --- .../contiguous_internalmem_datafacade.hpp | 72 +++++++++------ include/engine/datafacade/datafacade_base.hpp | 8 +- .../routing_algorithms/routing_base.hpp | 9 +- .../extractor/edge_based_graph_factory.hpp | 20 +++-- include/extractor/edge_based_node.hpp | 2 +- include/extractor/extractor_config.hpp | 2 + include/extractor/files.hpp | 30 +++++++ include/extractor/node_data_container.hpp | 83 ++++++++++++++++++ include/extractor/serialization.hpp | 27 ++++-- include/extractor/turn_data_container.hpp | 42 +++------ include/storage/shared_datatype.hpp | 10 ++- include/storage/storage_config.hpp | 1 + include/updater/updater_config.hpp | 2 + src/extractor/edge_based_graph_factory.cpp | 87 ++++++++++++------- src/extractor/extractor.cpp | 1 + src/storage/storage.cpp | 60 ++++++++----- src/storage/storage_config.cpp | 6 +- src/updater/updater.cpp | 29 +++++-- src/util/assert.cpp | 1 + unit_tests/mocks/mock_datafacade.hpp | 13 +-- 20 files changed, 355 insertions(+), 150 deletions(-) create mode 100644 include/extractor/node_data_container.hpp diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp index bc09386d5..1d7304127 100644 --- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp +++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp @@ -13,6 +13,7 @@ #include "extractor/datasources.hpp" #include "extractor/guidance/turn_instruction.hpp" #include "extractor/guidance/turn_lane_types.hpp" +#include "extractor/node_data_container.hpp" #include "extractor/packed_osm_ids.hpp" #include "extractor/profile_properties.hpp" #include "extractor/segment_data_container.hpp" @@ -223,6 +224,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade util::vector_view m_turn_duration_penalties; extractor::SegmentDataView segment_data; extractor::TurnDataView turn_data; + extractor::EdgeBasedNodeDataView edge_based_node_data; util::vector_view m_datasource_name_data; util::vector_view m_datasource_name_offsets; @@ -319,17 +321,34 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade data_layout.num_entries[storage::DataLayout::COORDINATE_LIST]); } - void InitializeEdgeInformationPointers(storage::DataLayout &layout, char *memory_ptr) + void InitializeEdgeBasedNodeDataInformationPointers(storage::DataLayout &layout, + char *memory_ptr) { auto via_geometry_list_ptr = - layout.GetBlockPtr(memory_ptr, storage::DataLayout::VIA_NODE_LIST); + layout.GetBlockPtr(memory_ptr, storage::DataLayout::GEOMETRY_ID_LIST); util::vector_view geometry_ids( - via_geometry_list_ptr, layout.num_entries[storage::DataLayout::VIA_NODE_LIST]); + via_geometry_list_ptr, layout.num_entries[storage::DataLayout::GEOMETRY_ID_LIST]); - const auto travel_mode_list_ptr = - layout.GetBlockPtr(memory_ptr, storage::DataLayout::TRAVEL_MODE); + const auto name_id_list_ptr = + layout.GetBlockPtr(memory_ptr, storage::DataLayout::NAME_ID_LIST); + util::vector_view name_ids(name_id_list_ptr, + layout.num_entries[storage::DataLayout::NAME_ID_LIST]); + + const auto travel_mode_list_ptr = layout.GetBlockPtr( + memory_ptr, storage::DataLayout::TRAVEL_MODE_LIST); util::vector_view travel_modes( - travel_mode_list_ptr, layout.num_entries[storage::DataLayout::TRAVEL_MODE]); + travel_mode_list_ptr, layout.num_entries[storage::DataLayout::TRAVEL_MODE_LIST]); + + edge_based_node_data = extractor::EdgeBasedNodeDataView( + std::move(geometry_ids), std::move(name_ids), std::move(travel_modes)); + } + + void InitializeEdgeInformationPointers(storage::DataLayout &layout, char *memory_ptr) + { + auto node_ids_list_ptr = + layout.GetBlockPtr(memory_ptr, storage::DataLayout::EDGE_BASED_NODE_ID_LIST); + util::vector_view node_ids( + node_ids_list_ptr, layout.num_entries[storage::DataLayout::EDGE_BASED_NODE_ID_LIST]); const auto lane_data_id_ptr = layout.GetBlockPtr(memory_ptr, storage::DataLayout::LANE_DATA_ID); @@ -342,11 +361,6 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade util::vector_view turn_instructions( turn_instruction_list_ptr, layout.num_entries[storage::DataLayout::TURN_INSTRUCTION]); - const auto name_id_list_ptr = - layout.GetBlockPtr(memory_ptr, storage::DataLayout::NAME_ID_LIST); - util::vector_view name_ids(name_id_list_ptr, - layout.num_entries[storage::DataLayout::NAME_ID_LIST]); - const auto entry_class_id_list_ptr = layout.GetBlockPtr(memory_ptr, storage::DataLayout::ENTRY_CLASSID); util::vector_view entry_class_ids( @@ -362,11 +376,9 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade util::vector_view post_turn_bearings( post_turn_bearing_ptr, layout.num_entries[storage::DataLayout::POST_TURN_BEARING]); - turn_data = extractor::TurnDataView(std::move(geometry_ids), - std::move(name_ids), + turn_data = extractor::TurnDataView(std::move(node_ids), std::move(turn_instructions), std::move(lane_data_ids), - std::move(travel_modes), std::move(entry_class_ids), std::move(pre_turn_bearings), std::move(post_turn_bearings)); @@ -509,6 +521,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade { InitializeChecksumPointer(data_layout, memory_block); InitializeNodeInformationPointers(data_layout, memory_block); + InitializeEdgeBasedNodeDataInformationPointers(data_layout, memory_block); InitializeEdgeInformationPointers(data_layout, memory_block); InitializeTurnPenalties(data_layout, memory_block); InitializeGeometryPointers(data_layout, memory_block); @@ -540,7 +553,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade return m_osmnodeid_list.at(id); } - virtual std::vector GetUncompressedForwardGeometry(const EdgeID id) const override final + std::vector GetUncompressedForwardGeometry(const EdgeID id) const override final { auto range = segment_data.GetForwardGeometry(id); @@ -599,11 +612,6 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade return std::vector{range.begin(), range.end()}; } - virtual GeometryID GetGeometryIndexForEdgeID(const EdgeID id) const override final - { - return turn_data.GetGeometryID(id); - } - virtual TurnPenalty GetWeightPenaltyForEdgeID(const unsigned id) const override final { BOOST_ASSERT(m_turn_weight_penalties.size() > id); @@ -616,17 +624,17 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade return m_turn_duration_penalties[id]; } + NodeID GetEdgeBasedNodeID(const EdgeID id) const override final + { + return turn_data.GetNodeID(id); + } + extractor::guidance::TurnInstruction GetTurnInstructionForEdgeID(const EdgeID id) const override final { return turn_data.GetTurnInstruction(id); } - extractor::TravelMode GetTravelModeForEdgeID(const EdgeID id) const override final - { - return turn_data.GetTravelMode(id); - } - std::vector GetEdgesInBox(const util::Coordinate south_west, const util::Coordinate north_east) const override final { @@ -744,9 +752,19 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade unsigned GetCheckSum() const override final { return m_check_sum; } - NameID GetNameIndexFromEdgeID(const EdgeID id) const override final + GeometryID GetGeometryIndex(const NodeID id) const override final { - return turn_data.GetNameID(id); + return edge_based_node_data.GetGeometryID(id); + } + + extractor::TravelMode GetTravelMode(const NodeID id) const override final + { + return edge_based_node_data.GetTravelMode(id); + } + + NameID GetNameIndex(const NodeID id) const override final + { + return edge_based_node_data.GetNameID(id); } StringView GetNameForID(const NameID id) const override final diff --git a/include/engine/datafacade/datafacade_base.hpp b/include/engine/datafacade/datafacade_base.hpp index 95c0b9549..33afac84c 100644 --- a/include/engine/datafacade/datafacade_base.hpp +++ b/include/engine/datafacade/datafacade_base.hpp @@ -51,7 +51,7 @@ class BaseDataFacade virtual OSMNodeID GetOSMNodeIDOfNode(const NodeID id) const = 0; - virtual GeometryID GetGeometryIndexForEdgeID(const EdgeID id) const = 0; + virtual GeometryID GetGeometryIndex(const NodeID id) const = 0; virtual std::vector GetUncompressedForwardGeometry(const EdgeID id) const = 0; @@ -79,10 +79,12 @@ class BaseDataFacade // Gets the name of a datasource virtual StringView GetDatasourceName(const DatasourceID id) const = 0; + virtual NodeID GetEdgeBasedNodeID(const NodeID id) const = 0; + virtual extractor::guidance::TurnInstruction GetTurnInstructionForEdgeID(const EdgeID id) const = 0; - virtual extractor::TravelMode GetTravelModeForEdgeID(const EdgeID id) const = 0; + virtual extractor::TravelMode GetTravelMode(const NodeID id) const = 0; virtual std::vector GetEdgesInBox(const util::Coordinate south_west, const util::Coordinate north_east) const = 0; @@ -135,7 +137,7 @@ class BaseDataFacade virtual extractor::guidance::TurnLaneDescription GetTurnDescription(const LaneDescriptionID lane_description_id) const = 0; - virtual NameID GetNameIndexFromEdgeID(const EdgeID id) const = 0; + virtual NameID GetNameIndex(const NodeID id) const = 0; virtual StringView GetNameForID(const NameID id) const = 0; diff --git a/include/engine/routing_algorithms/routing_base.hpp b/include/engine/routing_algorithms/routing_base.hpp index 681d13144..515560db6 100644 --- a/include/engine/routing_algorithms/routing_base.hpp +++ b/include/engine/routing_algorithms/routing_base.hpp @@ -115,15 +115,16 @@ void annotatePath(const FacadeT &facade, for (auto edge_id : unpacked_edges) { const auto &edge_data = facade.GetEdgeData(edge_id); - const auto turn_id = edge_data.turn_id; // edge-based node ID - const auto name_index = facade.GetNameIndexFromEdgeID(turn_id); + const auto turn_id = edge_data.turn_id; // edge-based edge ID + const auto ebg_node_id = facade.GetEdgeBasedNodeID(turn_id); // edge-based source node ID + const auto name_index = facade.GetNameIndex(ebg_node_id); const auto turn_instruction = facade.GetTurnInstructionForEdgeID(turn_id); const extractor::TravelMode travel_mode = (unpacked_path.empty() && start_traversed_in_reverse) ? phantom_node_pair.source_phantom.backward_travel_mode - : facade.GetTravelModeForEdgeID(turn_id); + : facade.GetTravelMode(ebg_node_id); - const auto geometry_index = facade.GetGeometryIndexForEdgeID(turn_id); + const auto geometry_index = facade.GetGeometryIndex(ebg_node_id); std::vector id_vector; std::vector weight_vector; diff --git a/include/extractor/edge_based_graph_factory.hpp b/include/extractor/edge_based_graph_factory.hpp index d2b90389c..438bd36f5 100644 --- a/include/extractor/edge_based_graph_factory.hpp +++ b/include/extractor/edge_based_graph_factory.hpp @@ -83,7 +83,8 @@ class EdgeBasedGraphFactory guidance::LaneDescriptionMap &lane_description_map); void Run(ScriptingEnvironment &scripting_environment, - const std::string &original_edge_data_filename, + const std::string &nodes_data_filename, + const std::string &turn_data_filename, const std::string &turn_lane_data_filename, const std::string &turn_weight_penalties_filename, const std::string &turn_duration_penalties_filename, @@ -151,12 +152,17 @@ class EdgeBasedGraphFactory std::vector GenerateEdgeExpandedNodes(); - void GenerateEdgeExpandedEdges(ScriptingEnvironment &scripting_environment, - const std::string &original_edge_data_filename, - const std::string &turn_lane_data_filename, - const std::string &turn_weight_penalties_filename, - const std::string &turn_duration_penalties_filename, - const std::string &turn_penalties_index_filename); + std::unordered_map + GenerateEdgeExpandedNodesData(const std::string &node_data_filename) const; + + void + GenerateEdgeExpandedEdges(ScriptingEnvironment &scripting_environment, + const std::unordered_map &index_nbg_edgeid_ebg_nodeid, + const std::string &original_edge_data_filename, + const std::string &turn_lane_data_filename, + const std::string &turn_weight_penalties_filename, + const std::string &turn_duration_penalties_filename, + const std::string &turn_penalties_index_filename); NBGToEBG InsertEdgeBasedNode(const NodeID u, const NodeID v); diff --git a/include/extractor/edge_based_node.hpp b/include/extractor/edge_based_node.hpp index bfd635b06..07d41aaeb 100644 --- a/include/extractor/edge_based_node.hpp +++ b/include/extractor/edge_based_node.hpp @@ -52,7 +52,7 @@ struct EdgeBasedNode SegmentID reverse_segment_id; // needed for edge-expanded graph NodeID u; // indices into the coordinates array NodeID v; // indices into the coordinates array - unsigned name_id; // id of the edge name + NameID name_id; // id of the edge name unsigned packed_geometry_id; struct diff --git a/include/extractor/extractor_config.hpp b/include/extractor/extractor_config.hpp index f47d4f55a..005334650 100644 --- a/include/extractor/extractor_config.hpp +++ b/include/extractor/extractor_config.hpp @@ -74,6 +74,7 @@ struct ExtractorConfig turn_weight_penalties_path = basepath + ".osrm.turn_weight_penalties"; turn_penalties_index_path = basepath + ".osrm.turn_penalties_index"; edge_based_node_weights_output_path = basepath + ".osrm.enw"; + edge_based_nodes_data_path = basepath + ".osrm.nodes_data"; profile_properties_output_path = basepath + ".osrm.properties"; intersection_class_data_output_path = basepath + ".osrm.icd"; compressed_node_based_graph_output_path = basepath + ".osrm.cnbg"; @@ -92,6 +93,7 @@ struct ExtractorConfig std::string geometry_output_path; std::string edge_output_path; std::string edge_graph_output_path; + std::string edge_based_nodes_data_path; std::string edge_based_node_weights_output_path; std::string node_output_path; std::string rtree_nodes_output_path; diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 32d434230..42d561163 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -2,7 +2,9 @@ #define OSRM_EXTRACTOR_FILES_HPP #include "extractor/guidance/turn_lane_types.hpp" +#include "extractor/node_data_container.hpp" #include "extractor/serialization.hpp" +#include "extractor/turn_data_container.hpp" #include "util/coordinate.hpp" #include "util/packed_vector.hpp" @@ -140,6 +142,34 @@ inline void writeTurnData(const boost::filesystem::path &path, const TurnDataT & serialization::write(writer, turn_data); } +// reads .osrm.nodes_data +template +inline void readNodeData(const boost::filesystem::path &path, NodeDataT &node_data) +{ + static_assert(std::is_same::value || + std::is_same::value || + std::is_same::value, + ""); + const auto fingerprint = storage::io::FileReader::VerifyFingerprint; + storage::io::FileReader reader{path, fingerprint}; + + serialization::read(reader, node_data); +} + +// writes .osrm.nodes_data +template +inline void writeNodeData(const boost::filesystem::path &path, const NodeDataT &node_data) +{ + static_assert(std::is_same::value || + std::is_same::value || + std::is_same::value, + ""); + const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; + storage::io::FileWriter writer{path, fingerprint}; + + serialization::write(writer, node_data); +} + // reads .osrm.tls template inline void readTurnLaneDescriptions(const boost::filesystem::path &path, diff --git a/include/extractor/node_data_container.hpp b/include/extractor/node_data_container.hpp new file mode 100644 index 000000000..0b00874bc --- /dev/null +++ b/include/extractor/node_data_container.hpp @@ -0,0 +1,83 @@ +#ifndef OSRM_EXTRACTOR_NODE_DATA_CONTAINER_HPP +#define OSRM_EXTRACTOR_NODE_DATA_CONTAINER_HPP + +#include "storage/io_fwd.hpp" +#include "storage/shared_memory_ownership.hpp" + +#include "util/typedefs.hpp" +#include "util/vector_view.hpp" + +namespace osrm +{ +namespace extractor +{ +namespace detail +{ +template class EdgeBasedNodeDataContainerImpl; +} + +namespace serialization +{ +template +void read(storage::io::FileReader &reader, + detail::EdgeBasedNodeDataContainerImpl &ebn_data); + +template +void write(storage::io::FileWriter &writer, + const detail::EdgeBasedNodeDataContainerImpl &ebn_data); +} + +namespace detail +{ +template class EdgeBasedNodeDataContainerImpl +{ + template using Vector = util::ViewOrVector; + + public: + EdgeBasedNodeDataContainerImpl() = default; + + EdgeBasedNodeDataContainerImpl(Vector geometry_ids, + Vector name_ids, + Vector travel_modes) + : geometry_ids(std::move(geometry_ids)), name_ids(std::move(name_ids)), + travel_modes(std::move(travel_modes)) + { + } + + GeometryID GetGeometryID(const NodeID id) const { return geometry_ids[id]; } + + extractor::TravelMode GetTravelMode(const NodeID id) const { return travel_modes[id]; } + + NameID GetNameID(const NodeID id) const { return name_ids[id]; } + + // Used by EdgeBasedGraphFactory to fill data structure + template > + void push_back(GeometryID geometry_id, NameID name_id, extractor::TravelMode travel_mode) + { + geometry_ids.push_back(geometry_id); + name_ids.push_back(name_id); + travel_modes.push_back(travel_mode); + } + + friend void serialization::read(storage::io::FileReader &reader, + EdgeBasedNodeDataContainerImpl &ebn_data_container); + friend void + serialization::write(storage::io::FileWriter &writer, + const EdgeBasedNodeDataContainerImpl &ebn_data_container); + + private: + Vector geometry_ids; + Vector name_ids; + Vector travel_modes; +}; +} + +using EdgeBasedNodeDataExternalContainer = + detail::EdgeBasedNodeDataContainerImpl; +using EdgeBasedNodeDataContainer = + detail::EdgeBasedNodeDataContainerImpl; +using EdgeBasedNodeDataView = detail::EdgeBasedNodeDataContainerImpl; +} +} + +#endif diff --git a/include/extractor/serialization.hpp b/include/extractor/serialization.hpp index 2f297ccdd..9d63f16c6 100644 --- a/include/extractor/serialization.hpp +++ b/include/extractor/serialization.hpp @@ -3,6 +3,7 @@ #include "extractor/datasources.hpp" #include "extractor/nbg_to_ebg.hpp" +#include "extractor/node_data_container.hpp" #include "extractor/segment_data_container.hpp" #include "extractor/turn_data_container.hpp" @@ -58,11 +59,9 @@ template inline void read(storage::io::FileReader &reader, detail::TurnDataContainerImpl &turn_data_container) { - storage::serialization::read(reader, turn_data_container.geometry_ids); - storage::serialization::read(reader, turn_data_container.name_ids); + storage::serialization::read(reader, turn_data_container.node_data_ids); storage::serialization::read(reader, turn_data_container.turn_instructions); storage::serialization::read(reader, turn_data_container.lane_data_ids); - storage::serialization::read(reader, turn_data_container.travel_modes); storage::serialization::read(reader, turn_data_container.entry_class_ids); storage::serialization::read(reader, turn_data_container.pre_turn_bearings); storage::serialization::read(reader, turn_data_container.post_turn_bearings); @@ -72,15 +71,31 @@ template inline void write(storage::io::FileWriter &writer, const detail::TurnDataContainerImpl &turn_data_container) { - storage::serialization::write(writer, turn_data_container.geometry_ids); - storage::serialization::write(writer, turn_data_container.name_ids); + storage::serialization::write(writer, turn_data_container.node_data_ids); storage::serialization::write(writer, turn_data_container.turn_instructions); storage::serialization::write(writer, turn_data_container.lane_data_ids); - storage::serialization::write(writer, turn_data_container.travel_modes); storage::serialization::write(writer, turn_data_container.entry_class_ids); storage::serialization::write(writer, turn_data_container.pre_turn_bearings); storage::serialization::write(writer, turn_data_container.post_turn_bearings); } + +template +inline void read(storage::io::FileReader &reader, + detail::EdgeBasedNodeDataContainerImpl &node_data_container) +{ + storage::serialization::read(reader, node_data_container.geometry_ids); + storage::serialization::read(reader, node_data_container.name_ids); + storage::serialization::read(reader, node_data_container.travel_modes); +} + +template +inline void write(storage::io::FileWriter &writer, + const detail::EdgeBasedNodeDataContainerImpl &node_data_container) +{ + storage::serialization::write(writer, node_data_container.geometry_ids); + storage::serialization::write(writer, node_data_container.name_ids); + storage::serialization::write(writer, node_data_container.travel_modes); +} } } } diff --git a/include/extractor/turn_data_container.hpp b/include/extractor/turn_data_container.hpp index ec2012a2d..b88505fd8 100644 --- a/include/extractor/turn_data_container.hpp +++ b/include/extractor/turn_data_container.hpp @@ -40,29 +40,23 @@ template class TurnDataContainerImpl public: TurnDataContainerImpl() = default; - TurnDataContainerImpl(Vector geometry_ids_, - Vector name_ids_, - Vector turn_instructions_, - Vector lane_data_ids_, - Vector travel_modes_, - Vector entry_class_ids_, - Vector pre_turn_bearings_, - Vector post_turn_bearings_) - : geometry_ids(std::move(geometry_ids_)), name_ids(std::move(name_ids_)), - turn_instructions(std::move(turn_instructions_)), - lane_data_ids(std::move(lane_data_ids_)), travel_modes(std::move(travel_modes_)), - entry_class_ids(std::move(entry_class_ids_)), - pre_turn_bearings(std::move(pre_turn_bearings_)), - post_turn_bearings(std::move(post_turn_bearings_)) + TurnDataContainerImpl(Vector node_data_ids, + Vector turn_instructions, + Vector lane_data_ids, + Vector entry_class_ids, + Vector pre_turn_bearings, + Vector post_turn_bearings) + : node_data_ids(std::move(node_data_ids)), turn_instructions(std::move(turn_instructions)), + lane_data_ids(std::move(lane_data_ids)), entry_class_ids(std::move(entry_class_ids)), + pre_turn_bearings(std::move(pre_turn_bearings)), + post_turn_bearings(std::move(post_turn_bearings)) { } - GeometryID GetGeometryID(const EdgeID id) const { return geometry_ids[id]; } + NodeID GetNodeID(const EdgeID id) const { return node_data_ids[id]; } EntryClassID GetEntryClassID(const EdgeID id) const { return entry_class_ids[id]; } - extractor::TravelMode GetTravelMode(const EdgeID id) const { return travel_modes[id]; } - util::guidance::TurnBearing GetPreTurnBearing(const EdgeID id) const { return pre_turn_bearings[id]; @@ -77,8 +71,6 @@ template class TurnDataContainerImpl bool HasLaneData(const EdgeID id) const { return INVALID_LANE_DATAID != lane_data_ids[id]; } - NameID GetNameID(const EdgeID id) const { return name_ids[id]; } - extractor::guidance::TurnInstruction GetTurnInstruction(const EdgeID id) const { return turn_instructions[id]; @@ -86,20 +78,16 @@ template class TurnDataContainerImpl // Used by EdgeBasedGraphFactory to fill data structure template > - void push_back(GeometryID geometry_id, - NameID name_id, + void push_back(NodeID node_data_id, extractor::guidance::TurnInstruction turn_instruction, LaneDataID lane_data_id, EntryClassID entry_class_id, - extractor::TravelMode travel_mode, util::guidance::TurnBearing pre_turn_bearing, util::guidance::TurnBearing post_turn_bearing) { - geometry_ids.push_back(geometry_id); - name_ids.push_back(name_id); + node_data_ids.push_back(node_data_id); turn_instructions.push_back(turn_instruction); lane_data_ids.push_back(lane_data_id); - travel_modes.push_back(travel_mode); entry_class_ids.push_back(entry_class_id); pre_turn_bearings.push_back(pre_turn_bearing); post_turn_bearings.push_back(post_turn_bearing); @@ -111,11 +99,9 @@ template class TurnDataContainerImpl const TurnDataContainerImpl &turn_data_container); private: - Vector geometry_ids; - Vector name_ids; + Vector node_data_ids; Vector turn_instructions; Vector lane_data_ids; - Vector travel_modes; Vector entry_class_ids; Vector pre_turn_bearings; Vector post_turn_bearings; diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp index 966d00938..f59ada642 100644 --- a/include/storage/shared_datatype.hpp +++ b/include/storage/shared_datatype.hpp @@ -19,14 +19,15 @@ namespace storage const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'}; const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA", + "GEOMETRY_ID_LIST", "NAME_ID_LIST", - "VIA_NODE_LIST", + "TRAVEL_MODE_LIST", "CH_GRAPH_NODE_LIST", "CH_GRAPH_EDGE_LIST", "COORDINATE_LIST", "OSM_NODE_ID_LIST", + "EDGE_BASED_NODE_ID_LIST", "TURN_INSTRUCTION", - "TRAVEL_MODE", "ENTRY_CLASSID", "R_SEARCH_TREE", "GEOMETRIES_INDEX", @@ -72,14 +73,15 @@ struct DataLayout enum BlockID { NAME_CHAR_DATA = 0, + GEOMETRY_ID_LIST, NAME_ID_LIST, - VIA_NODE_LIST, + TRAVEL_MODE_LIST, CH_GRAPH_NODE_LIST, CH_GRAPH_EDGE_LIST, COORDINATE_LIST, OSM_NODE_ID_LIST, + EDGE_BASED_NODE_ID_LIST, TURN_INSTRUCTION, - TRAVEL_MODE, ENTRY_CLASSID, R_SEARCH_TREE, GEOMETRIES_INDEX, diff --git a/include/storage/storage_config.hpp b/include/storage/storage_config.hpp index 9967eca92..fa71a0291 100644 --- a/include/storage/storage_config.hpp +++ b/include/storage/storage_config.hpp @@ -57,6 +57,7 @@ struct StorageConfig final boost::filesystem::path hsgr_data_path; boost::filesystem::path nodes_data_path; boost::filesystem::path edges_data_path; + boost::filesystem::path edge_based_nodes_data_path; boost::filesystem::path core_data_path; boost::filesystem::path geometries_path; boost::filesystem::path timestamp_path; diff --git a/include/updater/updater_config.hpp b/include/updater/updater_config.hpp index f5d84e541..27483b747 100644 --- a/include/updater/updater_config.hpp +++ b/include/updater/updater_config.hpp @@ -48,6 +48,7 @@ struct UpdaterConfig final turn_penalties_index_path = osrm_input_path.string() + ".turn_penalties_index"; node_based_graph_path = osrm_input_path.string() + ".nodes"; edge_data_path = osrm_input_path.string() + ".edges"; + edge_based_nodes_data_path = osrm_input_path.string() + ".nodes_data"; geometry_path = osrm_input_path.string() + ".geometry"; rtree_leaf_path = osrm_input_path.string() + ".fileIndex"; datasource_names_path = osrm_input_path.string() + ".datasource_names"; @@ -63,6 +64,7 @@ struct UpdaterConfig final std::string turn_penalties_index_path; std::string node_based_graph_path; std::string edge_data_path; + std::string edge_based_nodes_data_path; std::string geometry_path; std::string rtree_leaf_path; diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index 2af984db0..96fb384f9 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -168,6 +168,7 @@ NBGToEBG EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const N } void EdgeBasedGraphFactory::Run(ScriptingEnvironment &scripting_environment, + const std::string &nodes_data_filename, const std::string &turn_data_filename, const std::string &turn_lane_data_filename, const std::string &turn_weight_penalties_filename, @@ -186,8 +187,13 @@ void EdgeBasedGraphFactory::Run(ScriptingEnvironment &scripting_environment, } TIMER_STOP(generate_nodes); + TIMER_START(generate_nodes_data); + auto index_nbg_edgeid_to_ebg_nodeid = GenerateEdgeExpandedNodesData(nodes_data_filename); + TIMER_STOP(generate_nodes_data); + TIMER_START(generate_edges); GenerateEdgeExpandedEdges(scripting_environment, + index_nbg_edgeid_to_ebg_nodeid, turn_data_filename, turn_lane_data_filename, turn_weight_penalties_filename, @@ -292,9 +298,48 @@ std::vector EdgeBasedGraphFactory::GenerateEdgeExpandedNodes() return mapping; } +/// Actually it also generates turn data and serializes them... +std::unordered_map +EdgeBasedGraphFactory::GenerateEdgeExpandedNodesData(const std::string &node_data_filename) const +{ + // parallel_for( blocked_range( 0, n ), avg, auto_partitioner( ) ); + + NodeID ebg_node_id = 0; + std::unordered_map index; + EdgeBasedNodeDataExternalContainer node_data_container; + for (auto node_id : util::irange(0u, m_node_based_graph->GetNumberOfNodes())) + { + for (auto edge_id : m_node_based_graph->GetAdjacentEdgeRange(node_id)) + { + const EdgeData &edge_data = m_node_based_graph->GetEdgeData(edge_id); + const bool is_encoded_forwards = + m_compressed_edge_container.HasZippedEntryForForwardID(edge_id); + const bool is_encoded_backwards = + m_compressed_edge_container.HasZippedEntryForReverseID(edge_id); + if (is_encoded_forwards || is_encoded_backwards) + { + auto geometry_id = + is_encoded_forwards + ? m_compressed_edge_container.GetZippedPositionForForwardID(edge_id) + : m_compressed_edge_container.GetZippedPositionForReverseID(edge_id); + + BOOST_ASSERT(index.find(edge_id) == index.end()); + index.insert({edge_id, ebg_node_id++}); + node_data_container.push_back(GeometryID{geometry_id, is_encoded_forwards}, + edge_data.name_id, + edge_data.travel_mode); + } + } + } + + files::writeNodeData(node_data_filename, node_data_container); + return index; +} + /// Actually it also generates turn data and serializes them... void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( ScriptingEnvironment &scripting_environment, + const std::unordered_map &nbg_edgeid_to_ebg_nodeid, const std::string &turn_data_filename, const std::string &turn_lane_data_filename, const std::string &turn_weight_penalties_filename, @@ -454,39 +499,15 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( BOOST_ASSERT(!edge_data2.reversed); // the following is the core of the loop. - const bool is_encoded_forwards = - m_compressed_edge_container.HasZippedEntryForForwardID(incoming_edge); - const bool is_encoded_backwards = - m_compressed_edge_container.HasZippedEntryForReverseID(incoming_edge); - BOOST_ASSERT(is_encoded_forwards || is_encoded_backwards); - if (is_encoded_forwards) - { - turn_data_container.push_back( - GeometryID{m_compressed_edge_container.GetZippedPositionForForwardID( - incoming_edge), - true}, - edge_data1.name_id, - turn.instruction, - turn.lane_data_id, - entry_class_id, - edge_data1.travel_mode, - util::guidance::TurnBearing(intersection[0].bearing), - util::guidance::TurnBearing(turn.bearing)); - } - else if (is_encoded_backwards) - { - turn_data_container.push_back( - GeometryID{m_compressed_edge_container.GetZippedPositionForReverseID( - incoming_edge), - false}, - edge_data1.name_id, - turn.instruction, - turn.lane_data_id, - entry_class_id, - edge_data1.travel_mode, - util::guidance::TurnBearing(intersection[0].bearing), - util::guidance::TurnBearing(turn.bearing)); - } + BOOST_ASSERT(nbg_edgeid_to_ebg_nodeid.find(incoming_edge) != + nbg_edgeid_to_ebg_nodeid.end()); + turn_data_container.push_back( + nbg_edgeid_to_ebg_nodeid.find(incoming_edge)->second, + turn.instruction, + turn.lane_data_id, + entry_class_id, + util::guidance::TurnBearing(intersection[0].bearing), + util::guidance::TurnBearing(turn.bearing)); // compute weight and duration penalties auto is_traffic_light = m_traffic_lights.count(node_at_center_of_intersection); diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 6f1a41290..ba3fe5561 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -484,6 +484,7 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment, turn_lane_map); edge_based_graph_factory.Run(scripting_environment, + config.edge_based_nodes_data_path, config.edge_output_path, config.turn_lane_data_file_name, config.turn_weight_penalties_path, diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 3f138a78f..970406f8c 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -234,10 +234,7 @@ void Storage::PopulateLayout(DataLayout &layout) const auto number_of_original_edges = edges_file.ReadElementCount64(); // note: settings this all to the same size is correct, we extract them from the same struct - layout.SetBlockSize(DataLayout::VIA_NODE_LIST, number_of_original_edges); - layout.SetBlockSize(DataLayout::NAME_ID_LIST, number_of_original_edges); - layout.SetBlockSize(DataLayout::TRAVEL_MODE, - number_of_original_edges); + layout.SetBlockSize(DataLayout::EDGE_BASED_NODE_ID_LIST, number_of_original_edges); layout.SetBlockSize(DataLayout::PRE_TURN_BEARING, number_of_original_edges); layout.SetBlockSize(DataLayout::POST_TURN_BEARING, @@ -248,6 +245,16 @@ void Storage::PopulateLayout(DataLayout &layout) layout.SetBlockSize(DataLayout::ENTRY_CLASSID, number_of_original_edges); } + { + io::FileReader nodes_data_file(config.edge_based_nodes_data_path, + io::FileReader::VerifyFingerprint); + const auto nodes_number = nodes_data_file.ReadElementCount64(); + + layout.SetBlockSize(DataLayout::GEOMETRY_ID_LIST, nodes_number); + layout.SetBlockSize(DataLayout::NAME_ID_LIST, nodes_number); + layout.SetBlockSize(DataLayout::TRAVEL_MODE_LIST, nodes_number); + } + if (boost::filesystem::exists(config.hsgr_data_path)) { io::FileReader reader(config.hsgr_data_path, io::FileReader::VerifyFingerprint); @@ -573,17 +580,35 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) config.turn_lane_description_path, offsets, masks); } + // Load edge-based nodes data + { + auto geometry_id_list_ptr = + layout.GetBlockPtr(memory_ptr, storage::DataLayout::GEOMETRY_ID_LIST); + util::vector_view geometry_ids( + geometry_id_list_ptr, layout.num_entries[storage::DataLayout::GEOMETRY_ID_LIST]); + + auto name_id_list_ptr = + layout.GetBlockPtr(memory_ptr, storage::DataLayout::NAME_ID_LIST); + util::vector_view name_ids(name_id_list_ptr, + layout.num_entries[storage::DataLayout::NAME_ID_LIST]); + + auto travel_mode_list_ptr = layout.GetBlockPtr( + memory_ptr, storage::DataLayout::TRAVEL_MODE_LIST); + util::vector_view travel_modes( + travel_mode_list_ptr, layout.num_entries[storage::DataLayout::TRAVEL_MODE_LIST]); + + extractor::EdgeBasedNodeDataView node_data( + std::move(geometry_ids), std::move(name_ids), std::move(travel_modes)); + + extractor::files::readNodeData(config.edge_based_nodes_data_path, node_data); + } + // Load original edge data { - auto via_geometry_list_ptr = - layout.GetBlockPtr(memory_ptr, storage::DataLayout::VIA_NODE_LIST); - util::vector_view geometry_ids( - via_geometry_list_ptr, layout.num_entries[storage::DataLayout::VIA_NODE_LIST]); - - const auto travel_mode_list_ptr = layout.GetBlockPtr( - memory_ptr, storage::DataLayout::TRAVEL_MODE); - util::vector_view travel_modes( - travel_mode_list_ptr, layout.num_entries[storage::DataLayout::TRAVEL_MODE]); + auto node_id_list_ptr = layout.GetBlockPtr( + memory_ptr, storage::DataLayout::EDGE_BASED_NODE_ID_LIST); + util::vector_view node_ids( + node_id_list_ptr, layout.num_entries[storage::DataLayout::EDGE_BASED_NODE_ID_LIST]); const auto lane_data_id_ptr = layout.GetBlockPtr(memory_ptr, storage::DataLayout::LANE_DATA_ID); @@ -596,11 +621,6 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) util::vector_view turn_instructions( turn_instruction_list_ptr, layout.num_entries[storage::DataLayout::TURN_INSTRUCTION]); - const auto name_id_list_ptr = - layout.GetBlockPtr(memory_ptr, storage::DataLayout::NAME_ID_LIST); - util::vector_view name_ids(name_id_list_ptr, - layout.num_entries[storage::DataLayout::NAME_ID_LIST]); - const auto entry_class_id_list_ptr = layout.GetBlockPtr(memory_ptr, storage::DataLayout::ENTRY_CLASSID); util::vector_view entry_class_ids( @@ -616,11 +636,9 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) util::vector_view post_turn_bearings( post_turn_bearing_ptr, layout.num_entries[storage::DataLayout::POST_TURN_BEARING]); - extractor::TurnDataView turn_data(std::move(geometry_ids), - std::move(name_ids), + extractor::TurnDataView turn_data(std::move(node_ids), std::move(turn_instructions), std::move(lane_data_ids), - std::move(travel_modes), std::move(entry_class_ids), std::move(pre_turn_bearings), std::move(post_turn_bearings)); diff --git a/src/storage/storage_config.cpp b/src/storage/storage_config.cpp index bb2e40a53..2deca7681 100644 --- a/src/storage/storage_config.cpp +++ b/src/storage/storage_config.cpp @@ -27,8 +27,10 @@ bool CheckFileList(const std::vector &files) StorageConfig::StorageConfig(const boost::filesystem::path &base) : ram_index_path{base.string() + ".ramIndex"}, file_index_path{base.string() + ".fileIndex"}, hsgr_data_path{base.string() + ".hsgr"}, nodes_data_path{base.string() + ".nodes"}, - edges_data_path{base.string() + ".edges"}, core_data_path{base.string() + ".core"}, - geometries_path{base.string() + ".geometry"}, timestamp_path{base.string() + ".timestamp"}, + edges_data_path{base.string() + ".edges"}, + edge_based_nodes_data_path{base.string() + ".nodes_data"}, + core_data_path{base.string() + ".core"}, geometries_path{base.string() + ".geometry"}, + timestamp_path{base.string() + ".timestamp"}, turn_weight_penalties_path{base.string() + ".turn_weight_penalties"}, turn_duration_penalties_path{base.string() + ".turn_duration_penalties"}, datasource_names_path{base.string() + ".datasource_names"}, diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index d8b1d643a..e56efd148 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -76,12 +76,16 @@ void checkWeightsConsistency( extractor::SegmentDataContainer segment_data; extractor::files::readSegmentData(config.geometry_path, segment_data); + extractor::EdgeBasedNodeDataContainer node_data; + extractor::files::readNodeData(config.osrm_input_path.string() + ".nodes_data", node_data); + extractor::TurnDataContainer turn_data; extractor::files::readTurnData(config.osrm_input_path.string() + ".edges", turn_data); for (auto &edge : edge_based_edge_list) { - auto geometry_id = turn_data.GetGeometryID(edge.data.turn_id); + const auto node_id = turn_data.GetNodeID(edge.data.turn_id); + const auto geometry_id = node_data.GetGeometryID(node_id); if (geometry_id.forward) { @@ -457,6 +461,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e throw util::exception("Limit of 255 segment speed and turn penalty files each reached" + SOURCE_REF); + extractor::EdgeBasedNodeDataContainer node_data; extractor::TurnDataContainer turn_data; extractor::SegmentDataContainer segment_data; extractor::ProfileProperties profile_properties; @@ -468,6 +473,10 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e extractor::files::readSegmentData(config.geometry_path, segment_data); }; + const auto load_node_data = [&] { + extractor::files::readNodeData(config.edge_based_nodes_data_path, node_data); + }; + const auto load_edge_data = [&] { extractor::files::readTurnData(config.edge_data_path, turn_data); }; @@ -491,7 +500,8 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e profile_properties = profile_properties_file.ReadOne(); }; - tbb::parallel_invoke(load_edge_data, + tbb::parallel_invoke(load_node_data, + load_edge_data, load_segment_data, load_turn_weight_penalties, load_turn_duration_penalties, @@ -524,11 +534,13 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e updated_segments.resize(offset + updated_turn_penalties.size()); // we need to re-compute all edges that have updated turn penalties. // this marks it for re-computation - std::transform( - updated_turn_penalties.begin(), - updated_turn_penalties.end(), - updated_segments.begin() + offset, - [&turn_data](const std::uint64_t turn_id) { return turn_data.GetGeometryID(turn_id); }); + std::transform(updated_turn_penalties.begin(), + updated_turn_penalties.end(), + updated_segments.begin() + offset, + [&node_data, &turn_data](const std::uint64_t turn_id) { + const auto node_id = turn_data.GetNodeID(turn_id); + return node_data.GetGeometryID(node_id); + }); } tbb::parallel_sort(updated_segments.begin(), @@ -586,7 +598,8 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e }); const auto update_edge = [&](extractor::EdgeBasedEdge &edge) { - const auto geometry_id = turn_data.GetGeometryID(edge.data.turn_id); + const auto node_id = turn_data.GetNodeID(edge.data.turn_id); + const auto geometry_id = node_data.GetGeometryID(node_id); auto updated_iter = std::lower_bound(updated_segments.begin(), updated_segments.end(), geometry_id, diff --git a/src/util/assert.cpp b/src/util/assert.cpp index 1437c4df1..0e8ab79c5 100644 --- a/src/util/assert.cpp +++ b/src/util/assert.cpp @@ -15,6 +15,7 @@ namespace std::cerr << "[assert][" << tid << "] " << file << ":" << line << "\nin: " << function << ": " << expr << "\n" << msg; + std::terminate(); } } diff --git a/unit_tests/mocks/mock_datafacade.hpp b/unit_tests/mocks/mock_datafacade.hpp index b333a6774..4097b05e3 100644 --- a/unit_tests/mocks/mock_datafacade.hpp +++ b/unit_tests/mocks/mock_datafacade.hpp @@ -30,7 +30,7 @@ class MockBaseDataFacade : public engine::datafacade::BaseDataFacade } OSMNodeID GetOSMNodeIDOfNode(const NodeID /* id */) const override { return OSMNodeID{0}; } bool EdgeIsCompressed(const EdgeID /* id */) const { return false; } - GeometryID GetGeometryIndexForEdgeID(const EdgeID /* id */) const override + GeometryID GetGeometryIndex(const NodeID /* id */) const override { return GeometryID{SPECIAL_GEOMETRYID, false}; } @@ -85,10 +85,6 @@ class MockBaseDataFacade : public engine::datafacade::BaseDataFacade { return extractor::guidance::TurnInstruction::NO_TURN(); } - extractor::TravelMode GetTravelModeForEdgeID(const EdgeID /* id */) const override - { - return TRAVEL_MODE_INACCESSIBLE; - } std::vector GetEdgesInBox(const util::Coordinate /* south_west */, const util::Coordinate /*north_east */) const override { @@ -178,7 +174,12 @@ class MockBaseDataFacade : public engine::datafacade::BaseDataFacade unsigned GetCheckSum() const override { return 0; } - NameID GetNameIndexFromEdgeID(const EdgeID /* id */) const override { return 0; } + NodeID GetEdgeBasedNodeID(const NodeID /*id*/) const override { return 0; } + extractor::TravelMode GetTravelMode(const NodeID /* id */) const override + { + return TRAVEL_MODE_INACCESSIBLE; + } + NameID GetNameIndex(const NodeID /* id */) const override { return 0; } StringView GetNameForID(const NameID) const override final { return {}; } StringView GetRefForID(const NameID) const override final { return {}; }