Refactor processing_chain by splitting into sub functions

CMakeLists.txt

@@ -90,7 +90,7 @@ set(
 
 add_library(COORDINATE OBJECT ${CoordinateGlob})
 add_library(GITDESCRIPTION OBJECT util/git_sha.cpp)
-add_library(OSRM ${OSRMSources} $<TARGET_OBJECTS:ANGLE> $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:GITDESCRIPTION> $<TARGET_OBJECTS:FINGERPRINT> $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:LOGGER> $<TARGET_OBJECTS:PHANTOMNODE> $<TARGET_OBJECTS:EXCEPTION> $<TARGET_OBJECTS:MERCATOR>)
+add_library(OSRM ${OSRMSources} $<TARGET_OBJECTS:ANGLE> $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:GITDESCRIPTION> $<TARGET_OBJECTS:FINGERPRINT> $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:LOGGER> $<TARGET_OBJECTS:PHANTOMNODE> $<TARGET_OBJECTS:EXCEPTION> $<TARGET_OBJECTS:MERCATOR> $<TARGET_OBJECTS:IMPORT>)
 
 add_library(FINGERPRINT OBJECT util/fingerprint.cpp)
 add_dependencies(FINGERPRINT FingerPrintConfigure)

contractor/edge_based_graph_factory.cpp

@@ -40,21 +40,21 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <iomanip>
 #include <limits>
 
-EdgeBasedGraphFactory::EdgeBasedGraphFactory(
+EdgeBasedGraphFactory::EdgeBasedGraphFactory(std::shared_ptr<NodeBasedDynamicGraph> node_based_graph,
-    const std::shared_ptr<NodeBasedDynamicGraph> &node_based_graph,
+                                   std::shared_ptr<RestrictionMap> restriction_map,
-    std::unique_ptr<RestrictionMap> restriction_map,
+                                   std::unique_ptr<std::vector<NodeID>> barrier_node_list,
-    std::vector<NodeID> &barrier_node_list,
+                                   std::unique_ptr<std::vector<NodeID>> traffic_light_node_list,
-    std::vector<NodeID> &traffic_light_node_list,
+                                   const std::vector<QueryNode> &node_info_list,
-    std::vector<QueryNode> &node_info_list,
+                                   const SpeedProfileProperties &speed_profile)
-    SpeedProfileProperties &speed_profile)
     : speed_profile(speed_profile),
       m_number_of_edge_based_nodes(std::numeric_limits<unsigned>::max()),
-      m_node_info_list(node_info_list), m_node_based_graph(node_based_graph),
+      m_node_info_list(node_info_list),
+      m_node_based_graph(std::move(node_based_graph)),
       m_restriction_map(std::move(restriction_map)), max_id(0), removed_node_count(0)
 {
     // insert into unordered sets for fast lookup
-    m_barrier_nodes.insert(barrier_node_list.begin(), barrier_node_list.end());
+    m_barrier_nodes.insert(barrier_node_list->begin(), barrier_node_list->end());
-    m_traffic_lights.insert(traffic_light_node_list.begin(), traffic_light_node_list.end());
+    m_traffic_lights.insert(traffic_light_node_list->begin(), traffic_light_node_list->end());
 }
 
 void EdgeBasedGraphFactory::GetEdgeBasedEdges(DeallocatingVector<EdgeBasedEdge> &output_edge_list)
@@ -362,11 +362,11 @@ void EdgeBasedGraphFactory::CompressGeometry()
             // update any involved turn restrictions
             m_restriction_map->FixupStartingTurnRestriction(node_u, node_v, node_w);
             m_restriction_map->FixupArrivingTurnRestriction(node_u, node_v, node_w,
-                                                            m_node_based_graph);
+                                                            *m_node_based_graph);
 
             m_restriction_map->FixupStartingTurnRestriction(node_w, node_v, node_u);
             m_restriction_map->FixupArrivingTurnRestriction(node_w, node_v, node_u,
-                                                            m_node_based_graph);
+                                                            *m_node_based_graph);
 
             // store compressed geometry in container
             m_geometry_compressor.CompressEdge(
@@ -415,6 +415,7 @@ void EdgeBasedGraphFactory::RenumberEdges()
         for (const auto current_edge : m_node_based_graph->GetAdjacentEdgeRange(current_node))
         {
             EdgeData &edge_data = m_node_based_graph->GetEdgeData(current_edge);
+            // FIXME when does that happen? why can we skip here?
             if (!edge_data.forward)
             {
                 continue;

contractor/edge_based_graph_factory.hpp

@@ -59,12 +59,12 @@ class EdgeBasedGraphFactory
 
     struct SpeedProfileProperties;
 
-    explicit EdgeBasedGraphFactory(const std::shared_ptr<NodeBasedDynamicGraph> &node_based_graph,
+    explicit EdgeBasedGraphFactory(std::shared_ptr<NodeBasedDynamicGraph> node_based_graph,
-                                   std::unique_ptr<RestrictionMap> restricion_map,
+                                   std::shared_ptr<RestrictionMap> restricion_map,
-                                   std::vector<NodeID> &barrier_node_list,
+                                   std::unique_ptr<std::vector<NodeID>> barrier_node_list,
-                                   std::vector<NodeID> &traffic_light_node_list,
+                                   std::unique_ptr<std::vector<NodeID>> traffic_light_node_list,
-                                   std::vector<QueryNode> &node_info_list,
+                                   const std::vector<QueryNode> &node_info_list,
-                                   SpeedProfileProperties &speed_profile);
+                                   const SpeedProfileProperties &speed_profile);
 
     void Run(const std::string &original_edge_data_filename,
              const std::string &geometry_filename,
@@ -97,15 +97,16 @@ class EdgeBasedGraphFactory
 
     unsigned m_number_of_edge_based_nodes;
 
-    std::vector<QueryNode> m_node_info_list;
     std::vector<EdgeBasedNode> m_edge_based_node_list;
     DeallocatingVector<EdgeBasedEdge> m_edge_based_edge_list;
 
+    const std::vector<QueryNode>& m_node_info_list;
     std::shared_ptr<NodeBasedDynamicGraph> m_node_based_graph;
+    std::shared_ptr<RestrictionMap> m_restriction_map;
+
     std::unordered_set<NodeID> m_barrier_nodes;
     std::unordered_set<NodeID> m_traffic_lights;
 
-    std::unique_ptr<RestrictionMap> m_restriction_map;
 
     GeometryCompressor m_geometry_compressor;
 

contractor/processing_chain.cpp

@@ -63,17 +63,25 @@ Prepare::~Prepare() {}
 
 int Prepare::Process(int argc, char *argv[])
 {
+#ifdef WIN32
+#pragma message("Memory consumption on Windows can be higher due to different bit packing")
+#else
+    static_assert(sizeof(ImportEdge) == 20,
+                  "changing ImportEdge type has influence on memory consumption!");
+    static_assert(sizeof(EdgeBasedEdge) == 16,
+                  "changing EdgeBasedEdge type has influence on memory consumption!");
+#endif
+
     LogPolicy::GetInstance().Unmute();
     TIMER_START(preparing);
-    TIMER_START(expansion);
 
     if (!ParseArguments(argc, argv))
     {
         return 0;
     }
-    if (!boost::filesystem::is_regular_file(input_path))
+    if (!boost::filesystem::is_regular_file(osrm_input_path))
     {
-        SimpleLogger().Write(logWARNING) << "Input file " << input_path.string() << " not found!";
+        SimpleLogger().Write(logWARNING) << "Input file " << osrm_input_path.string() << " not found!";
         return 1;
     }
 
@@ -91,7 +99,7 @@ int Prepare::Process(int argc, char *argv[])
 
     const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
 
-    SimpleLogger().Write() << "Input file: " << input_path.filename().string();
+    SimpleLogger().Write() << "Input file: " << osrm_input_path.filename().string();
     SimpleLogger().Write() << "Restrictions file: " << restrictions_path.filename().string();
     SimpleLogger().Write() << "Profile: " << profile_path.filename().string();
     SimpleLogger().Write() << "Threads: " << requested_num_threads;
@@ -107,65 +115,40 @@ int Prepare::Process(int argc, char *argv[])
     LogPolicy::GetInstance().Unmute();
 
     FingerPrint fingerprint_orig;
-    CheckRestrictionsFile(fingerprint_orig);
 
-    boost::filesystem::ifstream input_stream(input_path, std::ios::in | std::ios::binary);
+    node_output_path = osrm_input_path.string() + ".nodes";
-
+    edge_output_path = osrm_input_path.string() + ".edges";
-    node_filename = input_path.string() + ".nodes";
+    geometry_output_path = osrm_input_path.string() + ".geometry";
-    edge_out = input_path.string() + ".edges";
+    graph_output_path = osrm_input_path.string() + ".hsgr";
-    geometry_filename = input_path.string() + ".geometry";
+    rtree_nodes_output_path = osrm_input_path.string() + ".ramIndex";
-    graph_out = input_path.string() + ".hsgr";
+    rtree_leafs_output_path = osrm_input_path.string() + ".fileIndex";
-    rtree_nodes_path = input_path.string() + ".ramIndex";
-    rtree_leafs_path = input_path.string() + ".fileIndex";
 
     /*** Setup Scripting Environment ***/
     // Create a new lua state
-    lua_State *lua_state = luaL_newstate();
 
-    // Connect LuaBind to this lua state
-    luabind::open(lua_state);
 
-    EdgeBasedGraphFactory::SpeedProfileProperties speed_profile;
+    SimpleLogger().Write() << "Generating edge-expanded graph representation";
 
-    if (!SetupScriptingEnvironment(lua_state, speed_profile))
+    TIMER_START(expansion);
-    {
-        return 1;
-    }
-
-#ifdef WIN32
-#pragma message("Memory consumption on Windows can be higher due to different bit packing")
-#else
-    static_assert(sizeof(ImportEdge) == 20,
-                  "changing ImportEdge type has influence on memory consumption!");
-#endif
-    NodeID number_of_node_based_nodes = readBinaryOSRMGraphFromStream(
-        input_stream, edge_list, barrier_node_list, traffic_light_list,
-        &internal_to_external_node_map, restriction_list);
-    input_stream.close();
-
-    if (edge_list.empty())
-    {
-        SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
-        return 1;
-    }
-
-    SimpleLogger().Write() << restriction_list.size() << " restrictions, "
-                           << barrier_node_list.size() << " bollard nodes, "
-                           << traffic_light_list.size() << " traffic lights";
 
     std::vector<EdgeBasedNode> node_based_edge_list;
-    unsigned number_of_edge_based_nodes = 0;
     DeallocatingVector<EdgeBasedEdge> edge_based_edge_list;
+    auto internal_to_external_node_map = osrm::make_unique<std::vector<QueryNode>>();
+    auto graph_size =
+        BuildEdgeExpandedGraph(*internal_to_external_node_map,
+                               node_based_edge_list, edge_based_edge_list);
 
-    // init node_based_edge_list, edge_based_edge_list by edgeList
+    auto number_of_node_based_nodes = graph_size.first;
-    number_of_edge_based_nodes =
+    auto number_of_edge_based_nodes = graph_size.second;
-        BuildEdgeExpandedGraph(lua_state, number_of_node_based_nodes, node_based_edge_list,
-                               edge_based_edge_list, speed_profile);
-    lua_close(lua_state);
 
     TIMER_STOP(expansion);
 
-    BuildRTree(node_based_edge_list);
+    SimpleLogger().Write() << "building r-tree ...";
+    TIMER_START(rtree);
+
+    BuildRTree(node_based_edge_list, *internal_to_external_node_map);
+
+    TIMER_STOP(rtree);
 
     RangebasedCRC32 crc32;
     if (crc32.using_hardware())
@@ -182,7 +165,8 @@ int Prepare::Process(int argc, char *argv[])
     node_based_edge_list.shrink_to_fit();
     SimpleLogger().Write() << "CRC32: " << crc32_value;
 
-    WriteNodeMapping();
+    SimpleLogger().Write() << "writing node map ...";
+    WriteNodeMapping(std::move(internal_to_external_node_map));
 
     /***
      * Contracting the edge-expanded graph
@@ -211,7 +195,7 @@ int Prepare::Process(int argc, char *argv[])
     SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count
                            << " edges";
 
-    boost::filesystem::ofstream hsgr_output_stream(graph_out, std::ios::binary);
+    boost::filesystem::ofstream hsgr_output_stream(graph_output_path, std::ios::binary);
     hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint));
     const unsigned max_used_node_id = 1 + [&contracted_edge_list]
     {
@@ -358,7 +342,7 @@ bool Prepare::ParseArguments(int argc, char *argv[])
     // shown to the user
     boost::program_options::options_description hidden_options("Hidden options");
     hidden_options.add_options()(
-        "input,i", boost::program_options::value<boost::filesystem::path>(&input_path),
+        "input,i", boost::program_options::value<boost::filesystem::path>(&osrm_input_path),
         "Input file in .osm, .osm.bz2 or .osm.pbf format");
 
     // positional option
@@ -408,7 +392,7 @@ bool Prepare::ParseArguments(int argc, char *argv[])
 
     if (!option_variables.count("restrictions"))
     {
-        restrictions_path = std::string(input_path.string() + ".restrictions");
+        restrictions_path = std::string(osrm_input_path.string() + ".restrictions");
     }
 
     if (!option_variables.count("input"))
@@ -420,36 +404,11 @@ bool Prepare::ParseArguments(int argc, char *argv[])
     return true;
 }
 
-/**
- \brief Loads and checks file UUIDs
-*/
-void Prepare::CheckRestrictionsFile(FingerPrint &fingerprint_orig)
-{
-    boost::filesystem::ifstream restriction_stream(restrictions_path, std::ios::binary);
-    FingerPrint fingerprint_loaded;
-    unsigned number_of_usable_restrictions = 0;
-    restriction_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint));
-    if (!fingerprint_loaded.TestPrepare(fingerprint_orig))
-    {
-        SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n"
-                                            "Reprocess to get rid of this warning.";
-    }
-
-    restriction_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned));
-    restriction_list.resize(number_of_usable_restrictions);
-    if (number_of_usable_restrictions > 0)
-    {
-        restriction_stream.read((char *)&(restriction_list[0]),
-                                number_of_usable_restrictions * sizeof(TurnRestriction));
-    }
-    restriction_stream.close();
-}
-
 /**
     \brief Setups scripting environment (lua-scripting)
     Also initializes speed profile.
 */
-bool Prepare::SetupScriptingEnvironment(
+void Prepare::SetupScriptingEnvironment(
     lua_State *lua_state, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile)
 {
     // open utility libraries string library;
@@ -461,14 +420,16 @@ bool Prepare::SetupScriptingEnvironment(
     // Now call our function in a lua script
     if (0 != luaL_dofile(lua_state, profile_path.string().c_str()))
     {
-        std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl;
+        std::stringstream msg;
-        return false;
+        msg << lua_tostring(lua_state, -1) << " occured in scripting block";
+        throw osrm::exception(msg.str());
     }
 
     if (0 != luaL_dostring(lua_state, "return traffic_signal_penalty\n"))
     {
-        std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl;
+        std::stringstream msg;
-        return false;
+        msg << lua_tostring(lua_state, -1) << " occured in scripting block";
+        throw osrm::exception(msg.str());
     }
     speed_profile.traffic_signal_penalty = 10 * lua_tointeger(lua_state, -1);
     SimpleLogger().Write(logDEBUG)
@@ -476,92 +437,134 @@ bool Prepare::SetupScriptingEnvironment(
 
     if (0 != luaL_dostring(lua_state, "return u_turn_penalty\n"))
     {
-        std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl;
+        std::stringstream msg;
-        return false;
+        msg << lua_tostring(lua_state, -1) << " occured in scripting block";
+        throw osrm::exception(msg.str());
     }
 
     speed_profile.u_turn_penalty = 10 * lua_tointeger(lua_state, -1);
     speed_profile.has_turn_penalty_function = lua_function_exists(lua_state, "turn_function");
-
-    return true;
 }
 
+/**
+  \brief Build load restrictions from .restriction file
+  */
+void Prepare::LoadRestrictionMap(const std::unordered_map<NodeID, NodeID> &external_to_internal_node_map,
+                                 RestrictionMap &restriction_map)
+{
+    boost::filesystem::ifstream input_stream(restrictions_path, std::ios::in | std::ios::binary);
+
+    std::vector<TurnRestriction> restriction_list;
+    loadRestrictionsFromFile(input_stream, external_to_internal_node_map, restriction_list);
+
+    SimpleLogger().Write() << " - " << restriction_list.size() << " restrictions.";
+
+    restriction_map = RestrictionMap(restriction_list);
+}
+
+/**
+  \brief Build load node based graph from .osrm file and restrictions from .restrictions file
+  */
+std::shared_ptr<NodeBasedDynamicGraph>
+Prepare::LoadNodeBasedGraph(std::vector<NodeID> &barrier_node_list,
+                            std::vector<NodeID> &traffic_light_list,
+                            RestrictionMap &restriction_map,
+                            std::vector<QueryNode>& internal_to_external_node_map)
+{
+    std::vector<ImportEdge> edge_list;
+    std::unordered_map<NodeID, NodeID> external_to_internal_node_map;
+
+    boost::filesystem::ifstream input_stream(osrm_input_path, std::ios::in | std::ios::binary);
+
+    NodeID number_of_node_based_nodes = loadNodesFromFile(input_stream,
+                                            barrier_node_list, traffic_light_list,
+                                            internal_to_external_node_map,
+                                            external_to_internal_node_map);
+
+    SimpleLogger().Write() << " - " << barrier_node_list.size() << " bollard nodes, "
+                           << traffic_light_list.size() << " traffic lights";
+
+    loadEdgesFromFile(input_stream, external_to_internal_node_map, edge_list);
+
+    LoadRestrictionMap(external_to_internal_node_map, restriction_map);
+
+    if (edge_list.empty())
+    {
+        SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
+        return std::shared_ptr<NodeBasedDynamicGraph>();
+    }
+
+    return NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list);
+}
+
+
 /**
  \brief Building an edge-expanded graph from node-based input and turn restrictions
 */
-std::size_t
+std::pair<std::size_t, std::size_t>
-Prepare::BuildEdgeExpandedGraph(lua_State *lua_state,
+Prepare::BuildEdgeExpandedGraph(std::vector<QueryNode> &internal_to_external_node_map,
-                                NodeID number_of_node_based_nodes,
                                 std::vector<EdgeBasedNode> &node_based_edge_list,
-                                DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list,
+                                DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list)
-                                EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile)
 {
-    SimpleLogger().Write() << "Generating edge-expanded graph representation";
+    lua_State *lua_state = luaL_newstate();
-    std::shared_ptr<NodeBasedDynamicGraph> node_based_graph =
+    luabind::open(lua_state);
-        NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list);
-    std::unique_ptr<RestrictionMap> restriction_map =
-        osrm::make_unique<RestrictionMap>(restriction_list);
-    std::shared_ptr<EdgeBasedGraphFactory> edge_based_graph_factory =
-        std::make_shared<EdgeBasedGraphFactory>(node_based_graph, std::move(restriction_map),
-                                                barrier_node_list, traffic_light_list,
-                                                internal_to_external_node_map, speed_profile);
-    edge_list.clear();
-    edge_list.shrink_to_fit();
 
-    edge_based_graph_factory->Run(edge_out, geometry_filename, lua_state);
+    EdgeBasedGraphFactory::SpeedProfileProperties speed_profile;
 
-    restriction_list.clear();
+    SetupScriptingEnvironment(lua_state, speed_profile);
-    restriction_list.shrink_to_fit();
+
-    barrier_node_list.clear();
+    auto barrier_node_list = osrm::make_unique<std::vector<NodeID>>();
-    barrier_node_list.shrink_to_fit();
+    auto traffic_light_list = osrm::make_unique<std::vector<NodeID>>();
-    traffic_light_list.clear();
+    auto restriction_map = std::make_shared<RestrictionMap>();
-    traffic_light_list.shrink_to_fit();
+
+    auto node_based_graph = LoadNodeBasedGraph(*barrier_node_list, *traffic_light_list, *restriction_map, internal_to_external_node_map);
+
+    const std::size_t number_of_node_based_nodes = node_based_graph->GetNumberOfNodes();
+
+    EdgeBasedGraphFactory edge_based_graph_factory(node_based_graph,
+                                                   restriction_map,
+                                                   std::move(barrier_node_list),
+                                                   std::move(traffic_light_list),
+                                                   internal_to_external_node_map,
+                                                   speed_profile);
+
+    edge_based_graph_factory.Run(edge_output_path, geometry_output_path, lua_state);
+    lua_close(lua_state);
 
     const std::size_t number_of_edge_based_nodes =
-        edge_based_graph_factory->GetNumberOfEdgeBasedNodes();
+        edge_based_graph_factory.GetNumberOfEdgeBasedNodes();
 
     BOOST_ASSERT(number_of_edge_based_nodes != std::numeric_limits<unsigned>::max());
-#ifndef WIN32
-    static_assert(sizeof(EdgeBasedEdge) == 16,
-                  "changing ImportEdge type has influence on memory consumption!");
-#endif
 
-    edge_based_graph_factory->GetEdgeBasedEdges(edge_based_edge_list);
+    edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list);
-    edge_based_graph_factory->GetEdgeBasedNodes(node_based_edge_list);
+    edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list);
 
-    edge_based_graph_factory.reset();
+    return std::make_pair(number_of_node_based_nodes, number_of_edge_based_nodes);
-    node_based_graph.reset();
-
-    return number_of_edge_based_nodes;
 }
 
 /**
   \brief Writing info on original (node-based) nodes
  */
-void Prepare::WriteNodeMapping()
+void Prepare::WriteNodeMapping(std::unique_ptr<std::vector<QueryNode>> internal_to_external_node_map)
 {
-    SimpleLogger().Write() << "writing node map ...";
+    boost::filesystem::ofstream node_stream(node_output_path, std::ios::binary);
-    boost::filesystem::ofstream node_stream(node_filename, std::ios::binary);
+    const unsigned size_of_mapping = internal_to_external_node_map->size();
-    const unsigned size_of_mapping = internal_to_external_node_map.size();
     node_stream.write((char *)&size_of_mapping, sizeof(unsigned));
     if (size_of_mapping > 0)
     {
-        node_stream.write((char *)&(internal_to_external_node_map[0]),
+        node_stream.write((char *) internal_to_external_node_map->data(),
                           size_of_mapping * sizeof(QueryNode));
     }
     node_stream.close();
-    internal_to_external_node_map.clear();
-    internal_to_external_node_map.shrink_to_fit();
 }
 
 /**
     \brief Building rtree-based nearest-neighbor data structure
 
-    Saves info to files: '.ramIndex' and '.fileIndex'.
+    Saves tree into '.ramIndex' and leaves into '.fileIndex'.
  */
-void Prepare::BuildRTree(std::vector<EdgeBasedNode> &node_based_edge_list)
+void Prepare::BuildRTree(const std::vector<EdgeBasedNode> &node_based_edge_list, const std::vector<QueryNode>& internal_to_external_node_map)
 {
-    SimpleLogger().Write() << "building r-tree ...";
+    StaticRTree<EdgeBasedNode>(node_based_edge_list, rtree_nodes_output_path.c_str(),
-    StaticRTree<EdgeBasedNode>(node_based_edge_list, rtree_nodes_path.c_str(),
+                               rtree_leafs_output_path.c_str(), internal_to_external_node_map);
-                               rtree_leafs_path.c_str(), internal_to_external_node_map);
 }

contractor/processing_chain.hpp

@@ -58,38 +58,38 @@ class Prepare
 
   protected:
     bool ParseArguments(int argc, char *argv[]);
-    void CheckRestrictionsFile(FingerPrint &fingerprint_orig);
+    void SetupScriptingEnvironment(lua_State *myLuaState,
-    bool SetupScriptingEnvironment(lua_State *myLuaState,
                                    EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile);
-    std::size_t BuildEdgeExpandedGraph(lua_State *myLuaState,
+    void LoadRestrictionMap(const std::unordered_map<NodeID, NodeID> &external_to_internal_node_map,
-                                       NodeID nodeBasedNodeNumber,
+                            RestrictionMap &restriction_map);
-                                       std::vector<EdgeBasedNode> &nodeBasedEdgeList,
+    std::shared_ptr<NodeBasedDynamicGraph>
-                                       DeallocatingVector<EdgeBasedEdge> &edgeBasedEdgeList,
+    LoadNodeBasedGraph(std::vector<NodeID> &barrier_node_list,
-                                       EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile);
+                       std::vector<NodeID> &traffic_light_list,
-    void WriteNodeMapping();
+                       RestrictionMap &restriction_map,
-    void BuildRTree(std::vector<EdgeBasedNode> &node_based_edge_list);
+                       std::vector<QueryNode>& internal_to_external_node_map);
+    std::pair<std::size_t, std::size_t>
+    BuildEdgeExpandedGraph(std::vector<QueryNode> &internal_to_external_node_map,
+                                       std::vector<EdgeBasedNode> &node_based_edge_list,
+                                       DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list);
+    void WriteNodeMapping(std::unique_ptr<std::vector<QueryNode>> internal_to_external_node_map);
+    void BuildRTree(const std::vector<EdgeBasedNode> &node_based_edge_list,
+                    const std::vector<QueryNode> &internal_to_external_node_map);
 
   private:
-    std::vector<QueryNode> internal_to_external_node_map;
-    std::vector<TurnRestriction> restriction_list;
-    std::vector<NodeID> barrier_node_list;
-    std::vector<NodeID> traffic_light_list;
-    std::vector<ImportEdge> edge_list;
 
     unsigned requested_num_threads;
     boost::filesystem::path config_file_path;
-    boost::filesystem::path input_path;
+    boost::filesystem::path osrm_input_path;
     boost::filesystem::path restrictions_path;
     boost::filesystem::path preinfo_path;
     boost::filesystem::path profile_path;
 
-    std::string node_filename;
+    std::string node_output_path;
-    std::string edge_out;
+    std::string edge_output_path;
-    std::string info_out;
+    std::string geometry_output_path;
-    std::string geometry_filename;
+    std::string graph_output_path;
-    std::string graph_out;
+    std::string rtree_nodes_output_path;
-    std::string rtree_nodes_path;
+    std::string rtree_leafs_output_path;
-    std::string rtree_leafs_path;
 };
 
 #endif // PROCESSING_CHAIN_HPP

data_structures/node_based_graph.hpp

@@ -186,9 +186,8 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector<ImportEdge
     SimpleLogger().Write() << "merged " << edges_list.size() - edge_count << " edges out of "
                            << edges_list.size();
 
-    auto graph = std::make_shared<NodeBasedDynamicGraph>(
+    return std::make_shared<NodeBasedDynamicGraph>(
         static_cast<NodeBasedDynamicGraph::NodeIterator>(number_of_nodes), edges_list);
-    return graph;
 }
 
 #endif // NODE_BASED_GRAPH_HPP

data_structures/restriction_map.hpp

@@ -91,6 +91,7 @@ template <> struct hash<RestrictionTarget>
 class RestrictionMap
 {
   public:
+    RestrictionMap() = default;
     RestrictionMap(const std::vector<TurnRestriction> &restriction_list);
 
     // Replace end v with w in each turn restriction containing u as via node
@@ -98,7 +99,7 @@ class RestrictionMap
     void FixupArrivingTurnRestriction(const NodeID node_u,
                                       const NodeID node_v,
                                       const NodeID node_w,
-                                      const std::shared_ptr<GraphT> &graph)
+                                      const GraphT &graph)
     {
         BOOST_ASSERT(node_u != SPECIAL_NODEID);
         BOOST_ASSERT(node_v != SPECIAL_NODEID);
@@ -112,9 +113,9 @@ class RestrictionMap
         // find all potential start edges. It is more efficent to get a (small) list
         // of potential start edges than iterating over all buckets
         std::vector<NodeID> predecessors;
-        for (const EdgeID current_edge_id : graph->GetAdjacentEdgeRange(node_u))
+        for (const EdgeID current_edge_id : graph.GetAdjacentEdgeRange(node_u))
         {
-            const NodeID target = graph->GetTarget(current_edge_id);
+            const NodeID target = graph.GetTarget(current_edge_id);
             if (node_v != target)
             {
                 predecessors.push_back(target);

data_structures/static_rtree.hpp

@@ -344,7 +344,7 @@ class StaticRTree
     StaticRTree(const StaticRTree &) = delete;
 
     // Construct a packed Hilbert-R-Tree with Kamel-Faloutsos algorithm [1]
-    explicit StaticRTree(std::vector<EdgeDataT> &input_data_vector,
+    explicit StaticRTree(const std::vector<EdgeDataT> &input_data_vector,
                          const std::string tree_node_filename,
                          const std::string leaf_node_filename,
                          const std::vector<QueryNode> &coordinate_list)

util/graph_loader.hpp

@@ -52,269 +52,34 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <unordered_map>
 #include <vector>
 
-template <typename EdgeT>
+/**
-NodeID read_undirected_osrm_stream(std::istream &input_stream,
+ * Reads the .restrictions file and loads it to a vector.
-                                   std::vector<EdgeT> &edge_list,
+ * The since the restrictions reference nodes using their external node id,
-                                   std::vector<FixedPointCoordinate> &coordinate_list)
+ * we need to renumber it to the new internal id.
+*/
+unsigned loadRestrictionsFromFile(std::istream& input_stream,
+                                  const std::unordered_map<NodeID, NodeID>& ext_to_int_id_map,
+                                  std::vector<TurnRestriction>& restriction_list)
 {
     const FingerPrint fingerprint_orig;
     FingerPrint fingerprint_loaded;
-    input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(FingerPrint));
+    unsigned number_of_usable_restrictions = 0;
-
+    input_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint));
-    if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig))
+    if (!fingerprint_loaded.TestPrepare(fingerprint_orig))
     {
-        SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n"
+        SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n"
                                             "Reprocess to get rid of this warning.";
     }
 
-    std::unordered_map<NodeID, NodeID> ext_to_int_id_map;
+    input_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned));
-
+    restriction_list.resize(number_of_usable_restrictions);
-    NodeID n;
+    if (number_of_usable_restrictions > 0)
-    input_stream.read(reinterpret_cast<char *>(&n), sizeof(NodeID));
-    SimpleLogger().Write() << "Importing n = " << n << " nodes ";
-
-    ExternalMemoryNode current_node;
-    for (NodeID i = 0; i < n; ++i)
     {
-        input_stream.read(reinterpret_cast<char *>(&current_node), sizeof(ExternalMemoryNode));
+        input_stream.read((char *) restriction_list.data(),
-        coordinate_list.emplace_back(current_node.lat, current_node.lon);
+                                number_of_usable_restrictions * sizeof(TurnRestriction));
-        ext_to_int_id_map.emplace(current_node.node_id, i);
-        // if (current_node.barrier)
-        // {
-        //     barrier_node_list.emplace_back(i);
-        // }
-        // if (current_node.traffic_lights)
-        // {
-        //     traffic_light_node_list.emplace_back(i);
-        // }
     }
 
-    // tighten vector sizes
+    // renumber ids referenced in restrictions
-    // barrier_node_list.shrink_to_fit();
-    // traffic_light_node_list.shrink_to_fit();
-
-    // renumber nodes in turn restrictions
-    // for (TurnRestriction &current_restriction : restriction_list)
-    // {
-    //     auto internal_id_iter = ext_to_int_id_map.find(current_restriction.from.node);
-    //     if (internal_id_iter == ext_to_int_id_map.end())
-    //     {
-    //         SimpleLogger().Write(logDEBUG) << "Unmapped from node " <<
-    //         current_restriction.from.node
-    //                                        << " of restriction";
-    //         continue;
-    //     }
-    //     current_restriction.from.node = internal_id_iter->second;
-
-    //     internal_id_iter = ext_to_int_id_map.find(current_restriction.via.node);
-    //     if (internal_id_iter == ext_to_int_id_map.end())
-    //     {
-    //         SimpleLogger().Write(logDEBUG) << "Unmapped via node " <<
-    //         current_restriction.via.node
-    //                                        << " of restriction";
-    //         continue;
-    //     }
-
-    //     current_restriction.via.node = internal_id_iter->second;
-
-    //     internal_id_iter = ext_to_int_id_map.find(current_restriction.to.node);
-    //     if (internal_id_iter == ext_to_int_id_map.end())
-    //     {
-    //         SimpleLogger().Write(logDEBUG) << "Unmapped to node " << current_restriction.to.node
-    //                                        << " of restriction";
-    //         continue;
-    //     }
-    //     current_restriction.to.node = internal_id_iter->second;
-    // }
-
-    EdgeWeight weight;
-    NodeID source, target;
-    unsigned nameID;
-    int length;
-    short dir; // direction (0 = open, 1 = forward, 2+ = open)
-    bool is_roundabout, ignore_in_grid, is_access_restricted, is_split;
-    TravelMode travel_mode;
-
-    EdgeID m;
-    input_stream.read(reinterpret_cast<char *>(&m), sizeof(unsigned));
-    edge_list.reserve(m);
-    SimpleLogger().Write() << " and " << m << " edges ";
-
-    for (EdgeID i = 0; i < m; ++i)
-    {
-        input_stream.read(reinterpret_cast<char *>(&source), sizeof(unsigned));
-        input_stream.read(reinterpret_cast<char *>(&target), sizeof(unsigned));
-        input_stream.read(reinterpret_cast<char *>(&length), sizeof(int));
-        input_stream.read(reinterpret_cast<char *>(&dir), sizeof(short));
-        input_stream.read(reinterpret_cast<char *>(&weight), sizeof(int));
-        input_stream.read(reinterpret_cast<char *>(&nameID), sizeof(unsigned));
-        input_stream.read(reinterpret_cast<char *>(&is_roundabout), sizeof(bool));
-        input_stream.read(reinterpret_cast<char *>(&ignore_in_grid), sizeof(bool));
-        input_stream.read(reinterpret_cast<char *>(&is_access_restricted), sizeof(bool));
-        input_stream.read(reinterpret_cast<char *>(&travel_mode), sizeof(TravelMode));
-        input_stream.read(reinterpret_cast<char *>(&is_split), sizeof(bool));
-
-        BOOST_ASSERT_MSG(length > 0, "loaded null length edge");
-        BOOST_ASSERT_MSG(weight > 0, "loaded null weight");
-        BOOST_ASSERT_MSG(0 <= dir && dir <= 2, "loaded bogus direction");
-
-        // bool forward = true;
-        // bool backward = true;
-        // if (1 == dir)
-        // {
-        //     backward = false;
-        // }
-        // if (2 == dir)
-        // {
-        //     forward = false;
-        // }
-
-        // translate the external NodeIDs to internal IDs
-        auto internal_id_iter = ext_to_int_id_map.find(source);
-        if (ext_to_int_id_map.find(source) == ext_to_int_id_map.end())
-        {
-#ifndef NDEBUG
-            SimpleLogger().Write(logWARNING) << " unresolved source NodeID: " << source;
-#endif
-            continue;
-        }
-        source = internal_id_iter->second;
-        internal_id_iter = ext_to_int_id_map.find(target);
-        if (ext_to_int_id_map.find(target) == ext_to_int_id_map.end())
-        {
-#ifndef NDEBUG
-            SimpleLogger().Write(logWARNING) << "unresolved target NodeID : " << target;
-#endif
-            continue;
-        }
-        target = internal_id_iter->second;
-        BOOST_ASSERT_MSG(source != SPECIAL_NODEID && target != SPECIAL_NODEID,
-                         "nonexisting source or target");
-
-        if (source > target)
-        {
-            std::swap(source, target);
-            // std::swap(forward, backward);
-        }
-
-        edge_list.emplace_back(source, target);
-    }
-    ext_to_int_id_map.clear();
-
-    tbb::parallel_sort(edge_list.begin(), edge_list.end());
-
-    // for (unsigned i = 1; i < edge_list.size(); ++i)
-    // {
-    //     if ((edge_list[i - 1].target == edge_list[i].target) &&
-    //         (edge_list[i - 1].source == edge_list[i].source))
-    //     {
-    //         const bool edge_flags_equivalent = (edge_list[i - 1].forward == edge_list[i].forward)
-    //         &&
-    //                                            (edge_list[i - 1].backward ==
-    //                                            edge_list[i].backward);
-    //         const bool edge_flags_are_superset1 =
-    //             (edge_list[i - 1].forward && edge_list[i - 1].backward) &&
-    //             (edge_list[i].forward != edge_list[i].backward);
-    //         const bool edge_flags_are_superset_2 =
-    //             (edge_list[i].forward && edge_list[i].backward) &&
-    //             (edge_list[i - 1].forward != edge_list[i - 1].backward);
-
-    //         if (edge_flags_equivalent)
-    //         {
-    //             edge_list[i].weight = std::min(edge_list[i - 1].weight, edge_list[i].weight);
-    //             edge_list[i - 1].source = SPECIAL_NODEID;
-    //         }
-    //         else if (edge_flags_are_superset1)
-    //         {
-    //             if (edge_list[i - 1].weight <= edge_list[i].weight)
-    //             {
-    //                 // edge i-1 is smaller and goes in both directions. Throw away the other edge
-    //                 edge_list[i].source = SPECIAL_NODEID;
-    //             }
-    //             else
-    //             {
-    //                 // edge i-1 is open in both directions, but edge i is smaller in one
-    //                 direction.
-    //                 // Close edge i-1 in this direction
-    //                 edge_list[i - 1].forward = !edge_list[i].forward;
-    //                 edge_list[i - 1].backward = !edge_list[i].backward;
-    //             }
-    //         }
-    //         else if (edge_flags_are_superset_2)
-    //         {
-    //             if (edge_list[i - 1].weight <= edge_list[i].weight)
-    //             {
-    //                 // edge i-1 is smaller for one direction. edge i is open in both. close edge
-    //                 i
-    //                 // in the other direction
-    //                 edge_list[i].forward = !edge_list[i - 1].forward;
-    //                 edge_list[i].backward = !edge_list[i - 1].backward;
-    //             }
-    //             else
-    //             {
-    //                 // edge i is smaller and goes in both direction. Throw away edge i-1
-    //                 edge_list[i - 1].source = SPECIAL_NODEID;
-    //             }
-    //         }
-    //     }
-    // }
-    // const auto new_end_iter =
-    //     std::remove_if(edge_list.begin(), edge_list.end(), [](const EdgeT &edge)
-    //                    {
-    //                        return edge.source == SPECIAL_NODEID || edge.target == SPECIAL_NODEID;
-    //                    });
-    // edge_list.erase(new_end_iter, edge_list.end()); // remove excess candidates.
-    // edge_list.shrink_to_fit();
-    SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges";
-    return n;
-}
-
-template <typename EdgeT>
-NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream,
-                                     std::vector<EdgeT> &edge_list,
-                                     std::vector<NodeID> &barrier_node_list,
-                                     std::vector<NodeID> &traffic_light_node_list,
-                                     std::vector<QueryNode> *int_to_ext_node_id_map,
-                                     std::vector<TurnRestriction> &restriction_list)
-{
-    const FingerPrint fingerprint_orig;
-    FingerPrint fingerprint_loaded;
-    input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(FingerPrint));
-
-    if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig))
-    {
-        SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n"
-                                            "Reprocess to get rid of this warning.";
-    }
-
-    std::unordered_map<NodeID, NodeID> ext_to_int_id_map;
-
-    NodeID n;
-    input_stream.read(reinterpret_cast<char *>(&n), sizeof(NodeID));
-    SimpleLogger().Write() << "Importing n = " << n << " nodes ";
-
-    ExternalMemoryNode current_node;
-    for (NodeID i = 0; i < n; ++i)
-    {
-        input_stream.read(reinterpret_cast<char *>(&current_node), sizeof(ExternalMemoryNode));
-        int_to_ext_node_id_map->emplace_back(current_node.lat, current_node.lon,
-                                             current_node.node_id);
-        ext_to_int_id_map.emplace(current_node.node_id, i);
-        if (current_node.barrier)
-        {
-            barrier_node_list.emplace_back(i);
-        }
-        if (current_node.traffic_lights)
-        {
-            traffic_light_node_list.emplace_back(i);
-        }
-    }
-
-    // tighten vector sizes
-    barrier_node_list.shrink_to_fit();
-    traffic_light_node_list.shrink_to_fit();
-
-    // renumber nodes in turn restrictions
     for (TurnRestriction &current_restriction : restriction_list)
     {
         auto internal_id_iter = ext_to_int_id_map.find(current_restriction.from.node);
@@ -346,6 +111,70 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream,
         current_restriction.to.node = internal_id_iter->second;
     }
 
+    return number_of_usable_restrictions;
+}
+
+
+/**
+ * Reads the beginning of an .osrm file and produces:
+ *  - list of barrier nodes
+ *  - list of traffic lights
+ *  - index to original node id map
+ *  - original node id to index map
+ */
+NodeID loadNodesFromFile(std::istream &input_stream,
+                         std::vector<NodeID> &barrier_node_list,
+                         std::vector<NodeID> &traffic_light_node_list,
+                         std::vector<QueryNode> &int_to_ext_node_id_map,
+                         std::unordered_map<NodeID, NodeID> &ext_to_int_id_map)
+{
+    const FingerPrint fingerprint_orig;
+    FingerPrint fingerprint_loaded;
+    input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(FingerPrint));
+
+    if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig))
+    {
+        SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n"
+                                            "Reprocess to get rid of this warning.";
+    }
+
+    NodeID n;
+    input_stream.read(reinterpret_cast<char *>(&n), sizeof(NodeID));
+    SimpleLogger().Write() << "Importing n = " << n << " nodes ";
+
+    ExternalMemoryNode current_node;
+    for (NodeID i = 0; i < n; ++i)
+    {
+        input_stream.read(reinterpret_cast<char *>(&current_node), sizeof(ExternalMemoryNode));
+        int_to_ext_node_id_map.emplace_back(current_node.lat, current_node.lon,
+                                             current_node.node_id);
+        ext_to_int_id_map.emplace(current_node.node_id, i);
+        if (current_node.barrier)
+        {
+            barrier_node_list.emplace_back(i);
+        }
+        if (current_node.traffic_lights)
+        {
+            traffic_light_node_list.emplace_back(i);
+        }
+    }
+
+    // tighten vector sizes
+    barrier_node_list.shrink_to_fit();
+    traffic_light_node_list.shrink_to_fit();
+
+    return n;
+}
+
+/**
+ * Reads a .osrm file and produces the edges. Edges reference nodes in the old
+ * OSM based format, we need to renumber it here.
+ */
+template <typename EdgeT>
+NodeID loadEdgesFromFile(std::istream &input_stream,
+                         const std::unordered_map<NodeID, NodeID> &ext_to_int_id_map,
+                         std::vector<EdgeT> &edge_list)
+{
     EdgeWeight weight;
     NodeID source, target;
     unsigned nameID;
@@ -419,10 +248,10 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream,
         edge_list.emplace_back(source, target, nameID, weight, forward, backward, is_roundabout,
                                ignore_in_grid, is_access_restricted, travel_mode, is_split);
     }
-    ext_to_int_id_map.clear();
 
     tbb::parallel_sort(edge_list.begin(), edge_list.end());
 
+    // Removes multi-edges between nodes
     for (unsigned i = 1; i < edge_list.size(); ++i)
     {
         if ((edge_list[i - 1].target == edge_list[i].target) &&
@@ -482,7 +311,8 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream,
     edge_list.erase(new_end_iter, edge_list.end()); // remove excess candidates.
     edge_list.shrink_to_fit();
     SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges";
-    return n;
+
+    return m;
 }
 
 template <typename NodeT, typename EdgeT>