From 1164a65df8eaca19fc5aecef03ed25c013cdc7cf Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 21 Apr 2015 10:43:02 +0200 Subject: [PATCH] Refactor processing_chain by splitting into sub functions --- CMakeLists.txt | 2 +- contractor/edge_based_graph_factory.cpp | 25 +- contractor/edge_based_graph_factory.hpp | 17 +- contractor/processing_chain.cpp | 269 ++++++++++--------- contractor/processing_chain.hpp | 44 +-- data_structures/node_based_graph.hpp | 3 +- data_structures/restriction_map.hpp | 7 +- data_structures/static_rtree.hpp | 2 +- util/graph_loader.hpp | 340 ++++++------------------ 9 files changed, 272 insertions(+), 437 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 503171a59..dd862d8dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,7 +90,7 @@ set( add_library(COORDINATE OBJECT ${CoordinateGlob}) add_library(GITDESCRIPTION OBJECT util/git_sha.cpp) -add_library(OSRM ${OSRMSources} $ $ $ $ $ $ $ $ $) +add_library(OSRM ${OSRMSources} $ $ $ $ $ $ $ $ $ $) add_library(FINGERPRINT OBJECT util/fingerprint.cpp) add_dependencies(FINGERPRINT FingerPrintConfigure) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index 268bb9f8d..241d43ec9 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -40,21 +40,21 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -EdgeBasedGraphFactory::EdgeBasedGraphFactory( - const std::shared_ptr &node_based_graph, - std::unique_ptr restriction_map, - std::vector &barrier_node_list, - std::vector &traffic_light_node_list, - std::vector &node_info_list, - SpeedProfileProperties &speed_profile) +EdgeBasedGraphFactory::EdgeBasedGraphFactory(std::shared_ptr node_based_graph, + std::shared_ptr restriction_map, + std::unique_ptr> barrier_node_list, + std::unique_ptr> traffic_light_node_list, + const std::vector &node_info_list, + const SpeedProfileProperties &speed_profile) : speed_profile(speed_profile), m_number_of_edge_based_nodes(std::numeric_limits::max()), - m_node_info_list(node_info_list), m_node_based_graph(node_based_graph), + m_node_info_list(node_info_list), + m_node_based_graph(std::move(node_based_graph)), m_restriction_map(std::move(restriction_map)), max_id(0), removed_node_count(0) { // insert into unordered sets for fast lookup - m_barrier_nodes.insert(barrier_node_list.begin(), barrier_node_list.end()); - m_traffic_lights.insert(traffic_light_node_list.begin(), traffic_light_node_list.end()); + m_barrier_nodes.insert(barrier_node_list->begin(), barrier_node_list->end()); + m_traffic_lights.insert(traffic_light_node_list->begin(), traffic_light_node_list->end()); } void EdgeBasedGraphFactory::GetEdgeBasedEdges(DeallocatingVector &output_edge_list) @@ -362,11 +362,11 @@ void EdgeBasedGraphFactory::CompressGeometry() // update any involved turn restrictions m_restriction_map->FixupStartingTurnRestriction(node_u, node_v, node_w); m_restriction_map->FixupArrivingTurnRestriction(node_u, node_v, node_w, - m_node_based_graph); + *m_node_based_graph); m_restriction_map->FixupStartingTurnRestriction(node_w, node_v, node_u); m_restriction_map->FixupArrivingTurnRestriction(node_w, node_v, node_u, - m_node_based_graph); + *m_node_based_graph); // store compressed geometry in container m_geometry_compressor.CompressEdge( @@ -415,6 +415,7 @@ void EdgeBasedGraphFactory::RenumberEdges() for (const auto current_edge : m_node_based_graph->GetAdjacentEdgeRange(current_node)) { EdgeData &edge_data = m_node_based_graph->GetEdgeData(current_edge); + // FIXME when does that happen? why can we skip here? if (!edge_data.forward) { continue; diff --git a/contractor/edge_based_graph_factory.hpp b/contractor/edge_based_graph_factory.hpp index 95b65babc..a5274396f 100644 --- a/contractor/edge_based_graph_factory.hpp +++ b/contractor/edge_based_graph_factory.hpp @@ -59,12 +59,12 @@ class EdgeBasedGraphFactory struct SpeedProfileProperties; - explicit EdgeBasedGraphFactory(const std::shared_ptr &node_based_graph, - std::unique_ptr restricion_map, - std::vector &barrier_node_list, - std::vector &traffic_light_node_list, - std::vector &node_info_list, - SpeedProfileProperties &speed_profile); + explicit EdgeBasedGraphFactory(std::shared_ptr node_based_graph, + std::shared_ptr restricion_map, + std::unique_ptr> barrier_node_list, + std::unique_ptr> traffic_light_node_list, + const std::vector &node_info_list, + const SpeedProfileProperties &speed_profile); void Run(const std::string &original_edge_data_filename, const std::string &geometry_filename, @@ -97,15 +97,16 @@ class EdgeBasedGraphFactory unsigned m_number_of_edge_based_nodes; - std::vector m_node_info_list; std::vector m_edge_based_node_list; DeallocatingVector m_edge_based_edge_list; + const std::vector& m_node_info_list; std::shared_ptr m_node_based_graph; + std::shared_ptr m_restriction_map; + std::unordered_set m_barrier_nodes; std::unordered_set m_traffic_lights; - std::unique_ptr m_restriction_map; GeometryCompressor m_geometry_compressor; diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index ccff7fd09..7ad682944 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -63,17 +63,25 @@ Prepare::~Prepare() {} int Prepare::Process(int argc, char *argv[]) { +#ifdef WIN32 +#pragma message("Memory consumption on Windows can be higher due to different bit packing") +#else + static_assert(sizeof(ImportEdge) == 20, + "changing ImportEdge type has influence on memory consumption!"); + static_assert(sizeof(EdgeBasedEdge) == 16, + "changing EdgeBasedEdge type has influence on memory consumption!"); +#endif + LogPolicy::GetInstance().Unmute(); TIMER_START(preparing); - TIMER_START(expansion); if (!ParseArguments(argc, argv)) { return 0; } - if (!boost::filesystem::is_regular_file(input_path)) + if (!boost::filesystem::is_regular_file(osrm_input_path)) { - SimpleLogger().Write(logWARNING) << "Input file " << input_path.string() << " not found!"; + SimpleLogger().Write(logWARNING) << "Input file " << osrm_input_path.string() << " not found!"; return 1; } @@ -91,7 +99,7 @@ int Prepare::Process(int argc, char *argv[]) const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); - SimpleLogger().Write() << "Input file: " << input_path.filename().string(); + SimpleLogger().Write() << "Input file: " << osrm_input_path.filename().string(); SimpleLogger().Write() << "Restrictions file: " << restrictions_path.filename().string(); SimpleLogger().Write() << "Profile: " << profile_path.filename().string(); SimpleLogger().Write() << "Threads: " << requested_num_threads; @@ -107,65 +115,40 @@ int Prepare::Process(int argc, char *argv[]) LogPolicy::GetInstance().Unmute(); FingerPrint fingerprint_orig; - CheckRestrictionsFile(fingerprint_orig); - boost::filesystem::ifstream input_stream(input_path, std::ios::in | std::ios::binary); - - node_filename = input_path.string() + ".nodes"; - edge_out = input_path.string() + ".edges"; - geometry_filename = input_path.string() + ".geometry"; - graph_out = input_path.string() + ".hsgr"; - rtree_nodes_path = input_path.string() + ".ramIndex"; - rtree_leafs_path = input_path.string() + ".fileIndex"; + node_output_path = osrm_input_path.string() + ".nodes"; + edge_output_path = osrm_input_path.string() + ".edges"; + geometry_output_path = osrm_input_path.string() + ".geometry"; + graph_output_path = osrm_input_path.string() + ".hsgr"; + rtree_nodes_output_path = osrm_input_path.string() + ".ramIndex"; + rtree_leafs_output_path = osrm_input_path.string() + ".fileIndex"; /*** Setup Scripting Environment ***/ // Create a new lua state - lua_State *lua_state = luaL_newstate(); - // Connect LuaBind to this lua state - luabind::open(lua_state); - EdgeBasedGraphFactory::SpeedProfileProperties speed_profile; + SimpleLogger().Write() << "Generating edge-expanded graph representation"; - if (!SetupScriptingEnvironment(lua_state, speed_profile)) - { - return 1; - } - -#ifdef WIN32 -#pragma message("Memory consumption on Windows can be higher due to different bit packing") -#else - static_assert(sizeof(ImportEdge) == 20, - "changing ImportEdge type has influence on memory consumption!"); -#endif - NodeID number_of_node_based_nodes = readBinaryOSRMGraphFromStream( - input_stream, edge_list, barrier_node_list, traffic_light_list, - &internal_to_external_node_map, restriction_list); - input_stream.close(); - - if (edge_list.empty()) - { - SimpleLogger().Write(logWARNING) << "The input data is empty, exiting."; - return 1; - } - - SimpleLogger().Write() << restriction_list.size() << " restrictions, " - << barrier_node_list.size() << " bollard nodes, " - << traffic_light_list.size() << " traffic lights"; + TIMER_START(expansion); std::vector node_based_edge_list; - unsigned number_of_edge_based_nodes = 0; DeallocatingVector edge_based_edge_list; + auto internal_to_external_node_map = osrm::make_unique>(); + auto graph_size = + BuildEdgeExpandedGraph(*internal_to_external_node_map, + node_based_edge_list, edge_based_edge_list); - // init node_based_edge_list, edge_based_edge_list by edgeList - number_of_edge_based_nodes = - BuildEdgeExpandedGraph(lua_state, number_of_node_based_nodes, node_based_edge_list, - edge_based_edge_list, speed_profile); - lua_close(lua_state); + auto number_of_node_based_nodes = graph_size.first; + auto number_of_edge_based_nodes = graph_size.second; TIMER_STOP(expansion); - BuildRTree(node_based_edge_list); + SimpleLogger().Write() << "building r-tree ..."; + TIMER_START(rtree); + + BuildRTree(node_based_edge_list, *internal_to_external_node_map); + + TIMER_STOP(rtree); RangebasedCRC32 crc32; if (crc32.using_hardware()) @@ -182,7 +165,8 @@ int Prepare::Process(int argc, char *argv[]) node_based_edge_list.shrink_to_fit(); SimpleLogger().Write() << "CRC32: " << crc32_value; - WriteNodeMapping(); + SimpleLogger().Write() << "writing node map ..."; + WriteNodeMapping(std::move(internal_to_external_node_map)); /*** * Contracting the edge-expanded graph @@ -211,7 +195,7 @@ int Prepare::Process(int argc, char *argv[]) SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count << " edges"; - boost::filesystem::ofstream hsgr_output_stream(graph_out, std::ios::binary); + boost::filesystem::ofstream hsgr_output_stream(graph_output_path, std::ios::binary); hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint)); const unsigned max_used_node_id = 1 + [&contracted_edge_list] { @@ -358,7 +342,7 @@ bool Prepare::ParseArguments(int argc, char *argv[]) // shown to the user boost::program_options::options_description hidden_options("Hidden options"); hidden_options.add_options()( - "input,i", boost::program_options::value(&input_path), + "input,i", boost::program_options::value(&osrm_input_path), "Input file in .osm, .osm.bz2 or .osm.pbf format"); // positional option @@ -408,7 +392,7 @@ bool Prepare::ParseArguments(int argc, char *argv[]) if (!option_variables.count("restrictions")) { - restrictions_path = std::string(input_path.string() + ".restrictions"); + restrictions_path = std::string(osrm_input_path.string() + ".restrictions"); } if (!option_variables.count("input")) @@ -420,36 +404,11 @@ bool Prepare::ParseArguments(int argc, char *argv[]) return true; } -/** - \brief Loads and checks file UUIDs -*/ -void Prepare::CheckRestrictionsFile(FingerPrint &fingerprint_orig) -{ - boost::filesystem::ifstream restriction_stream(restrictions_path, std::ios::binary); - FingerPrint fingerprint_loaded; - unsigned number_of_usable_restrictions = 0; - restriction_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint)); - if (!fingerprint_loaded.TestPrepare(fingerprint_orig)) - { - SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n" - "Reprocess to get rid of this warning."; - } - - restriction_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned)); - restriction_list.resize(number_of_usable_restrictions); - if (number_of_usable_restrictions > 0) - { - restriction_stream.read((char *)&(restriction_list[0]), - number_of_usable_restrictions * sizeof(TurnRestriction)); - } - restriction_stream.close(); -} - /** \brief Setups scripting environment (lua-scripting) Also initializes speed profile. */ -bool Prepare::SetupScriptingEnvironment( +void Prepare::SetupScriptingEnvironment( lua_State *lua_state, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile) { // open utility libraries string library; @@ -461,14 +420,16 @@ bool Prepare::SetupScriptingEnvironment( // Now call our function in a lua script if (0 != luaL_dofile(lua_state, profile_path.string().c_str())) { - std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; - return false; + std::stringstream msg; + msg << lua_tostring(lua_state, -1) << " occured in scripting block"; + throw osrm::exception(msg.str()); } if (0 != luaL_dostring(lua_state, "return traffic_signal_penalty\n")) { - std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; - return false; + std::stringstream msg; + msg << lua_tostring(lua_state, -1) << " occured in scripting block"; + throw osrm::exception(msg.str()); } speed_profile.traffic_signal_penalty = 10 * lua_tointeger(lua_state, -1); SimpleLogger().Write(logDEBUG) @@ -476,92 +437,134 @@ bool Prepare::SetupScriptingEnvironment( if (0 != luaL_dostring(lua_state, "return u_turn_penalty\n")) { - std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; - return false; + std::stringstream msg; + msg << lua_tostring(lua_state, -1) << " occured in scripting block"; + throw osrm::exception(msg.str()); } speed_profile.u_turn_penalty = 10 * lua_tointeger(lua_state, -1); speed_profile.has_turn_penalty_function = lua_function_exists(lua_state, "turn_function"); - - return true; } +/** + \brief Build load restrictions from .restriction file + */ +void Prepare::LoadRestrictionMap(const std::unordered_map &external_to_internal_node_map, + RestrictionMap &restriction_map) +{ + boost::filesystem::ifstream input_stream(restrictions_path, std::ios::in | std::ios::binary); + + std::vector restriction_list; + loadRestrictionsFromFile(input_stream, external_to_internal_node_map, restriction_list); + + SimpleLogger().Write() << " - " << restriction_list.size() << " restrictions."; + + restriction_map = RestrictionMap(restriction_list); +} + +/** + \brief Build load node based graph from .osrm file and restrictions from .restrictions file + */ +std::shared_ptr +Prepare::LoadNodeBasedGraph(std::vector &barrier_node_list, + std::vector &traffic_light_list, + RestrictionMap &restriction_map, + std::vector& internal_to_external_node_map) +{ + std::vector edge_list; + std::unordered_map external_to_internal_node_map; + + boost::filesystem::ifstream input_stream(osrm_input_path, std::ios::in | std::ios::binary); + + NodeID number_of_node_based_nodes = loadNodesFromFile(input_stream, + barrier_node_list, traffic_light_list, + internal_to_external_node_map, + external_to_internal_node_map); + + SimpleLogger().Write() << " - " << barrier_node_list.size() << " bollard nodes, " + << traffic_light_list.size() << " traffic lights"; + + loadEdgesFromFile(input_stream, external_to_internal_node_map, edge_list); + + LoadRestrictionMap(external_to_internal_node_map, restriction_map); + + if (edge_list.empty()) + { + SimpleLogger().Write(logWARNING) << "The input data is empty, exiting."; + return std::shared_ptr(); + } + + return NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list); +} + + /** \brief Building an edge-expanded graph from node-based input and turn restrictions */ -std::size_t -Prepare::BuildEdgeExpandedGraph(lua_State *lua_state, - NodeID number_of_node_based_nodes, +std::pair +Prepare::BuildEdgeExpandedGraph(std::vector &internal_to_external_node_map, std::vector &node_based_edge_list, - DeallocatingVector &edge_based_edge_list, - EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile) + DeallocatingVector &edge_based_edge_list) { - SimpleLogger().Write() << "Generating edge-expanded graph representation"; - std::shared_ptr node_based_graph = - NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list); - std::unique_ptr restriction_map = - osrm::make_unique(restriction_list); - std::shared_ptr edge_based_graph_factory = - std::make_shared(node_based_graph, std::move(restriction_map), - barrier_node_list, traffic_light_list, - internal_to_external_node_map, speed_profile); - edge_list.clear(); - edge_list.shrink_to_fit(); + lua_State *lua_state = luaL_newstate(); + luabind::open(lua_state); - edge_based_graph_factory->Run(edge_out, geometry_filename, lua_state); + EdgeBasedGraphFactory::SpeedProfileProperties speed_profile; - restriction_list.clear(); - restriction_list.shrink_to_fit(); - barrier_node_list.clear(); - barrier_node_list.shrink_to_fit(); - traffic_light_list.clear(); - traffic_light_list.shrink_to_fit(); + SetupScriptingEnvironment(lua_state, speed_profile); + + auto barrier_node_list = osrm::make_unique>(); + auto traffic_light_list = osrm::make_unique>(); + auto restriction_map = std::make_shared(); + + auto node_based_graph = LoadNodeBasedGraph(*barrier_node_list, *traffic_light_list, *restriction_map, internal_to_external_node_map); + + const std::size_t number_of_node_based_nodes = node_based_graph->GetNumberOfNodes(); + + EdgeBasedGraphFactory edge_based_graph_factory(node_based_graph, + restriction_map, + std::move(barrier_node_list), + std::move(traffic_light_list), + internal_to_external_node_map, + speed_profile); + + edge_based_graph_factory.Run(edge_output_path, geometry_output_path, lua_state); + lua_close(lua_state); const std::size_t number_of_edge_based_nodes = - edge_based_graph_factory->GetNumberOfEdgeBasedNodes(); + edge_based_graph_factory.GetNumberOfEdgeBasedNodes(); BOOST_ASSERT(number_of_edge_based_nodes != std::numeric_limits::max()); -#ifndef WIN32 - static_assert(sizeof(EdgeBasedEdge) == 16, - "changing ImportEdge type has influence on memory consumption!"); -#endif - edge_based_graph_factory->GetEdgeBasedEdges(edge_based_edge_list); - edge_based_graph_factory->GetEdgeBasedNodes(node_based_edge_list); + edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list); + edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list); - edge_based_graph_factory.reset(); - node_based_graph.reset(); - - return number_of_edge_based_nodes; + return std::make_pair(number_of_node_based_nodes, number_of_edge_based_nodes); } /** \brief Writing info on original (node-based) nodes */ -void Prepare::WriteNodeMapping() +void Prepare::WriteNodeMapping(std::unique_ptr> internal_to_external_node_map) { - SimpleLogger().Write() << "writing node map ..."; - boost::filesystem::ofstream node_stream(node_filename, std::ios::binary); - const unsigned size_of_mapping = internal_to_external_node_map.size(); + boost::filesystem::ofstream node_stream(node_output_path, std::ios::binary); + const unsigned size_of_mapping = internal_to_external_node_map->size(); node_stream.write((char *)&size_of_mapping, sizeof(unsigned)); if (size_of_mapping > 0) { - node_stream.write((char *)&(internal_to_external_node_map[0]), + node_stream.write((char *) internal_to_external_node_map->data(), size_of_mapping * sizeof(QueryNode)); } node_stream.close(); - internal_to_external_node_map.clear(); - internal_to_external_node_map.shrink_to_fit(); } /** \brief Building rtree-based nearest-neighbor data structure - Saves info to files: '.ramIndex' and '.fileIndex'. + Saves tree into '.ramIndex' and leaves into '.fileIndex'. */ -void Prepare::BuildRTree(std::vector &node_based_edge_list) +void Prepare::BuildRTree(const std::vector &node_based_edge_list, const std::vector& internal_to_external_node_map) { - SimpleLogger().Write() << "building r-tree ..."; - StaticRTree(node_based_edge_list, rtree_nodes_path.c_str(), - rtree_leafs_path.c_str(), internal_to_external_node_map); + StaticRTree(node_based_edge_list, rtree_nodes_output_path.c_str(), + rtree_leafs_output_path.c_str(), internal_to_external_node_map); } diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index 933213a4c..9d8f08f9a 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -58,38 +58,38 @@ class Prepare protected: bool ParseArguments(int argc, char *argv[]); - void CheckRestrictionsFile(FingerPrint &fingerprint_orig); - bool SetupScriptingEnvironment(lua_State *myLuaState, + void SetupScriptingEnvironment(lua_State *myLuaState, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile); - std::size_t BuildEdgeExpandedGraph(lua_State *myLuaState, - NodeID nodeBasedNodeNumber, - std::vector &nodeBasedEdgeList, - DeallocatingVector &edgeBasedEdgeList, - EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile); - void WriteNodeMapping(); - void BuildRTree(std::vector &node_based_edge_list); + void LoadRestrictionMap(const std::unordered_map &external_to_internal_node_map, + RestrictionMap &restriction_map); + std::shared_ptr + LoadNodeBasedGraph(std::vector &barrier_node_list, + std::vector &traffic_light_list, + RestrictionMap &restriction_map, + std::vector& internal_to_external_node_map); + std::pair + BuildEdgeExpandedGraph(std::vector &internal_to_external_node_map, + std::vector &node_based_edge_list, + DeallocatingVector &edge_based_edge_list); + void WriteNodeMapping(std::unique_ptr> internal_to_external_node_map); + void BuildRTree(const std::vector &node_based_edge_list, + const std::vector &internal_to_external_node_map); private: - std::vector internal_to_external_node_map; - std::vector restriction_list; - std::vector barrier_node_list; - std::vector traffic_light_list; - std::vector edge_list; unsigned requested_num_threads; boost::filesystem::path config_file_path; - boost::filesystem::path input_path; + boost::filesystem::path osrm_input_path; boost::filesystem::path restrictions_path; boost::filesystem::path preinfo_path; boost::filesystem::path profile_path; - std::string node_filename; - std::string edge_out; - std::string info_out; - std::string geometry_filename; - std::string graph_out; - std::string rtree_nodes_path; - std::string rtree_leafs_path; + std::string node_output_path; + std::string edge_output_path; + std::string geometry_output_path; + std::string graph_output_path; + std::string rtree_nodes_output_path; + std::string rtree_leafs_output_path; }; #endif // PROCESSING_CHAIN_HPP diff --git a/data_structures/node_based_graph.hpp b/data_structures/node_based_graph.hpp index bd71362c1..e3a0b594f 100644 --- a/data_structures/node_based_graph.hpp +++ b/data_structures/node_based_graph.hpp @@ -186,9 +186,8 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector( + return std::make_shared( static_cast(number_of_nodes), edges_list); - return graph; } #endif // NODE_BASED_GRAPH_HPP diff --git a/data_structures/restriction_map.hpp b/data_structures/restriction_map.hpp index 27a6698db..e2a89a008 100644 --- a/data_structures/restriction_map.hpp +++ b/data_structures/restriction_map.hpp @@ -91,6 +91,7 @@ template <> struct hash class RestrictionMap { public: + RestrictionMap() = default; RestrictionMap(const std::vector &restriction_list); // Replace end v with w in each turn restriction containing u as via node @@ -98,7 +99,7 @@ class RestrictionMap void FixupArrivingTurnRestriction(const NodeID node_u, const NodeID node_v, const NodeID node_w, - const std::shared_ptr &graph) + const GraphT &graph) { BOOST_ASSERT(node_u != SPECIAL_NODEID); BOOST_ASSERT(node_v != SPECIAL_NODEID); @@ -112,9 +113,9 @@ class RestrictionMap // find all potential start edges. It is more efficent to get a (small) list // of potential start edges than iterating over all buckets std::vector predecessors; - for (const EdgeID current_edge_id : graph->GetAdjacentEdgeRange(node_u)) + for (const EdgeID current_edge_id : graph.GetAdjacentEdgeRange(node_u)) { - const NodeID target = graph->GetTarget(current_edge_id); + const NodeID target = graph.GetTarget(current_edge_id); if (node_v != target) { predecessors.push_back(target); diff --git a/data_structures/static_rtree.hpp b/data_structures/static_rtree.hpp index 48ba20120..fa5607198 100644 --- a/data_structures/static_rtree.hpp +++ b/data_structures/static_rtree.hpp @@ -344,7 +344,7 @@ class StaticRTree StaticRTree(const StaticRTree &) = delete; // Construct a packed Hilbert-R-Tree with Kamel-Faloutsos algorithm [1] - explicit StaticRTree(std::vector &input_data_vector, + explicit StaticRTree(const std::vector &input_data_vector, const std::string tree_node_filename, const std::string leaf_node_filename, const std::vector &coordinate_list) diff --git a/util/graph_loader.hpp b/util/graph_loader.hpp index d741334ba..3779f0b9e 100644 --- a/util/graph_loader.hpp +++ b/util/graph_loader.hpp @@ -52,269 +52,34 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -template -NodeID read_undirected_osrm_stream(std::istream &input_stream, - std::vector &edge_list, - std::vector &coordinate_list) +/** + * Reads the .restrictions file and loads it to a vector. + * The since the restrictions reference nodes using their external node id, + * we need to renumber it to the new internal id. +*/ +unsigned loadRestrictionsFromFile(std::istream& input_stream, + const std::unordered_map& ext_to_int_id_map, + std::vector& restriction_list) { const FingerPrint fingerprint_orig; FingerPrint fingerprint_loaded; - input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(FingerPrint)); - - if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) + unsigned number_of_usable_restrictions = 0; + input_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint)); + if (!fingerprint_loaded.TestPrepare(fingerprint_orig)) { - SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n" + SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n" "Reprocess to get rid of this warning."; } - std::unordered_map ext_to_int_id_map; - - NodeID n; - input_stream.read(reinterpret_cast(&n), sizeof(NodeID)); - SimpleLogger().Write() << "Importing n = " << n << " nodes "; - - ExternalMemoryNode current_node; - for (NodeID i = 0; i < n; ++i) + input_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned)); + restriction_list.resize(number_of_usable_restrictions); + if (number_of_usable_restrictions > 0) { - input_stream.read(reinterpret_cast(¤t_node), sizeof(ExternalMemoryNode)); - coordinate_list.emplace_back(current_node.lat, current_node.lon); - ext_to_int_id_map.emplace(current_node.node_id, i); - // if (current_node.barrier) - // { - // barrier_node_list.emplace_back(i); - // } - // if (current_node.traffic_lights) - // { - // traffic_light_node_list.emplace_back(i); - // } + input_stream.read((char *) restriction_list.data(), + number_of_usable_restrictions * sizeof(TurnRestriction)); } - // tighten vector sizes - // barrier_node_list.shrink_to_fit(); - // traffic_light_node_list.shrink_to_fit(); - - // renumber nodes in turn restrictions - // for (TurnRestriction ¤t_restriction : restriction_list) - // { - // auto internal_id_iter = ext_to_int_id_map.find(current_restriction.from.node); - // if (internal_id_iter == ext_to_int_id_map.end()) - // { - // SimpleLogger().Write(logDEBUG) << "Unmapped from node " << - // current_restriction.from.node - // << " of restriction"; - // continue; - // } - // current_restriction.from.node = internal_id_iter->second; - - // internal_id_iter = ext_to_int_id_map.find(current_restriction.via.node); - // if (internal_id_iter == ext_to_int_id_map.end()) - // { - // SimpleLogger().Write(logDEBUG) << "Unmapped via node " << - // current_restriction.via.node - // << " of restriction"; - // continue; - // } - - // current_restriction.via.node = internal_id_iter->second; - - // internal_id_iter = ext_to_int_id_map.find(current_restriction.to.node); - // if (internal_id_iter == ext_to_int_id_map.end()) - // { - // SimpleLogger().Write(logDEBUG) << "Unmapped to node " << current_restriction.to.node - // << " of restriction"; - // continue; - // } - // current_restriction.to.node = internal_id_iter->second; - // } - - EdgeWeight weight; - NodeID source, target; - unsigned nameID; - int length; - short dir; // direction (0 = open, 1 = forward, 2+ = open) - bool is_roundabout, ignore_in_grid, is_access_restricted, is_split; - TravelMode travel_mode; - - EdgeID m; - input_stream.read(reinterpret_cast(&m), sizeof(unsigned)); - edge_list.reserve(m); - SimpleLogger().Write() << " and " << m << " edges "; - - for (EdgeID i = 0; i < m; ++i) - { - input_stream.read(reinterpret_cast(&source), sizeof(unsigned)); - input_stream.read(reinterpret_cast(&target), sizeof(unsigned)); - input_stream.read(reinterpret_cast(&length), sizeof(int)); - input_stream.read(reinterpret_cast(&dir), sizeof(short)); - input_stream.read(reinterpret_cast(&weight), sizeof(int)); - input_stream.read(reinterpret_cast(&nameID), sizeof(unsigned)); - input_stream.read(reinterpret_cast(&is_roundabout), sizeof(bool)); - input_stream.read(reinterpret_cast(&ignore_in_grid), sizeof(bool)); - input_stream.read(reinterpret_cast(&is_access_restricted), sizeof(bool)); - input_stream.read(reinterpret_cast(&travel_mode), sizeof(TravelMode)); - input_stream.read(reinterpret_cast(&is_split), sizeof(bool)); - - BOOST_ASSERT_MSG(length > 0, "loaded null length edge"); - BOOST_ASSERT_MSG(weight > 0, "loaded null weight"); - BOOST_ASSERT_MSG(0 <= dir && dir <= 2, "loaded bogus direction"); - - // bool forward = true; - // bool backward = true; - // if (1 == dir) - // { - // backward = false; - // } - // if (2 == dir) - // { - // forward = false; - // } - - // translate the external NodeIDs to internal IDs - auto internal_id_iter = ext_to_int_id_map.find(source); - if (ext_to_int_id_map.find(source) == ext_to_int_id_map.end()) - { -#ifndef NDEBUG - SimpleLogger().Write(logWARNING) << " unresolved source NodeID: " << source; -#endif - continue; - } - source = internal_id_iter->second; - internal_id_iter = ext_to_int_id_map.find(target); - if (ext_to_int_id_map.find(target) == ext_to_int_id_map.end()) - { -#ifndef NDEBUG - SimpleLogger().Write(logWARNING) << "unresolved target NodeID : " << target; -#endif - continue; - } - target = internal_id_iter->second; - BOOST_ASSERT_MSG(source != SPECIAL_NODEID && target != SPECIAL_NODEID, - "nonexisting source or target"); - - if (source > target) - { - std::swap(source, target); - // std::swap(forward, backward); - } - - edge_list.emplace_back(source, target); - } - ext_to_int_id_map.clear(); - - tbb::parallel_sort(edge_list.begin(), edge_list.end()); - - // for (unsigned i = 1; i < edge_list.size(); ++i) - // { - // if ((edge_list[i - 1].target == edge_list[i].target) && - // (edge_list[i - 1].source == edge_list[i].source)) - // { - // const bool edge_flags_equivalent = (edge_list[i - 1].forward == edge_list[i].forward) - // && - // (edge_list[i - 1].backward == - // edge_list[i].backward); - // const bool edge_flags_are_superset1 = - // (edge_list[i - 1].forward && edge_list[i - 1].backward) && - // (edge_list[i].forward != edge_list[i].backward); - // const bool edge_flags_are_superset_2 = - // (edge_list[i].forward && edge_list[i].backward) && - // (edge_list[i - 1].forward != edge_list[i - 1].backward); - - // if (edge_flags_equivalent) - // { - // edge_list[i].weight = std::min(edge_list[i - 1].weight, edge_list[i].weight); - // edge_list[i - 1].source = SPECIAL_NODEID; - // } - // else if (edge_flags_are_superset1) - // { - // if (edge_list[i - 1].weight <= edge_list[i].weight) - // { - // // edge i-1 is smaller and goes in both directions. Throw away the other edge - // edge_list[i].source = SPECIAL_NODEID; - // } - // else - // { - // // edge i-1 is open in both directions, but edge i is smaller in one - // direction. - // // Close edge i-1 in this direction - // edge_list[i - 1].forward = !edge_list[i].forward; - // edge_list[i - 1].backward = !edge_list[i].backward; - // } - // } - // else if (edge_flags_are_superset_2) - // { - // if (edge_list[i - 1].weight <= edge_list[i].weight) - // { - // // edge i-1 is smaller for one direction. edge i is open in both. close edge - // i - // // in the other direction - // edge_list[i].forward = !edge_list[i - 1].forward; - // edge_list[i].backward = !edge_list[i - 1].backward; - // } - // else - // { - // // edge i is smaller and goes in both direction. Throw away edge i-1 - // edge_list[i - 1].source = SPECIAL_NODEID; - // } - // } - // } - // } - // const auto new_end_iter = - // std::remove_if(edge_list.begin(), edge_list.end(), [](const EdgeT &edge) - // { - // return edge.source == SPECIAL_NODEID || edge.target == SPECIAL_NODEID; - // }); - // edge_list.erase(new_end_iter, edge_list.end()); // remove excess candidates. - // edge_list.shrink_to_fit(); - SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges"; - return n; -} - -template -NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream, - std::vector &edge_list, - std::vector &barrier_node_list, - std::vector &traffic_light_node_list, - std::vector *int_to_ext_node_id_map, - std::vector &restriction_list) -{ - const FingerPrint fingerprint_orig; - FingerPrint fingerprint_loaded; - input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(FingerPrint)); - - if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) - { - SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n" - "Reprocess to get rid of this warning."; - } - - std::unordered_map ext_to_int_id_map; - - NodeID n; - input_stream.read(reinterpret_cast(&n), sizeof(NodeID)); - SimpleLogger().Write() << "Importing n = " << n << " nodes "; - - ExternalMemoryNode current_node; - for (NodeID i = 0; i < n; ++i) - { - input_stream.read(reinterpret_cast(¤t_node), sizeof(ExternalMemoryNode)); - int_to_ext_node_id_map->emplace_back(current_node.lat, current_node.lon, - current_node.node_id); - ext_to_int_id_map.emplace(current_node.node_id, i); - if (current_node.barrier) - { - barrier_node_list.emplace_back(i); - } - if (current_node.traffic_lights) - { - traffic_light_node_list.emplace_back(i); - } - } - - // tighten vector sizes - barrier_node_list.shrink_to_fit(); - traffic_light_node_list.shrink_to_fit(); - - // renumber nodes in turn restrictions + // renumber ids referenced in restrictions for (TurnRestriction ¤t_restriction : restriction_list) { auto internal_id_iter = ext_to_int_id_map.find(current_restriction.from.node); @@ -346,6 +111,70 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream, current_restriction.to.node = internal_id_iter->second; } + return number_of_usable_restrictions; +} + + +/** + * Reads the beginning of an .osrm file and produces: + * - list of barrier nodes + * - list of traffic lights + * - index to original node id map + * - original node id to index map + */ +NodeID loadNodesFromFile(std::istream &input_stream, + std::vector &barrier_node_list, + std::vector &traffic_light_node_list, + std::vector &int_to_ext_node_id_map, + std::unordered_map &ext_to_int_id_map) +{ + const FingerPrint fingerprint_orig; + FingerPrint fingerprint_loaded; + input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(FingerPrint)); + + if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) + { + SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n" + "Reprocess to get rid of this warning."; + } + + NodeID n; + input_stream.read(reinterpret_cast(&n), sizeof(NodeID)); + SimpleLogger().Write() << "Importing n = " << n << " nodes "; + + ExternalMemoryNode current_node; + for (NodeID i = 0; i < n; ++i) + { + input_stream.read(reinterpret_cast(¤t_node), sizeof(ExternalMemoryNode)); + int_to_ext_node_id_map.emplace_back(current_node.lat, current_node.lon, + current_node.node_id); + ext_to_int_id_map.emplace(current_node.node_id, i); + if (current_node.barrier) + { + barrier_node_list.emplace_back(i); + } + if (current_node.traffic_lights) + { + traffic_light_node_list.emplace_back(i); + } + } + + // tighten vector sizes + barrier_node_list.shrink_to_fit(); + traffic_light_node_list.shrink_to_fit(); + + return n; +} + +/** + * Reads a .osrm file and produces the edges. Edges reference nodes in the old + * OSM based format, we need to renumber it here. + */ +template +NodeID loadEdgesFromFile(std::istream &input_stream, + const std::unordered_map &ext_to_int_id_map, + std::vector &edge_list) +{ EdgeWeight weight; NodeID source, target; unsigned nameID; @@ -419,10 +248,10 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream, edge_list.emplace_back(source, target, nameID, weight, forward, backward, is_roundabout, ignore_in_grid, is_access_restricted, travel_mode, is_split); } - ext_to_int_id_map.clear(); tbb::parallel_sort(edge_list.begin(), edge_list.end()); + // Removes multi-edges between nodes for (unsigned i = 1; i < edge_list.size(); ++i) { if ((edge_list[i - 1].target == edge_list[i].target) && @@ -482,7 +311,8 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream, edge_list.erase(new_end_iter, edge_list.end()); // remove excess candidates. edge_list.shrink_to_fit(); SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges"; - return n; + + return m; } template