Avoids Shuffling Data Around for Nothing, closes 3306

include/util/graph_loader.hpp

@@ -20,6 +20,7 @@
 
 #include <fstream>
 #include <ios>
+#include <unordered_set>
 #include <vector>
 
 namespace osrm
@@ -58,13 +59,14 @@ inline unsigned loadRestrictionsFromFile(std::istream &input_stream,
 
 /**
  * Reads the beginning of an .osrm file and produces:
- *  - list of barrier nodes
+ *  - barrier nodes
- *  - list of traffic lights
+ *  - traffic lights
  *  - nodes indexed by their internal (non-osm) id
  */
-inline NodeID loadNodesFromFile(std::istream &input_stream,
+template <typename BarrierOutIter, typename TrafficSignalsOutIter>
-                                std::vector<NodeID> &barrier_node_list,
+NodeID loadNodesFromFile(std::istream &input_stream,
-                                std::vector<NodeID> &traffic_light_node_list,
+                         BarrierOutIter barriers,
+                         TrafficSignalsOutIter traffic_signals,
                          std::vector<extractor::QueryNode> &node_array)
 {
     const FingerPrint fingerprint_valid = FingerPrint::GetValid();
@@ -89,19 +91,19 @@ inline NodeID loadNodesFromFile(std::istream &input_stream,
         input_stream.read(reinterpret_cast<char *>(&current_node),
                           sizeof(extractor::ExternalMemoryNode));
         node_array.emplace_back(current_node.lon, current_node.lat, current_node.node_id);
+
         if (current_node.barrier)
         {
-            barrier_node_list.emplace_back(i);
+            *barriers = i;
-        }
+            ++barriers;
-        if (current_node.traffic_lights)
-        {
-            traffic_light_node_list.emplace_back(i);
-        }
         }
 
-    // tighten vector sizes
+        if (current_node.traffic_lights)
-    barrier_node_list.shrink_to_fit();
+        {
-    traffic_light_node_list.shrink_to_fit();
+            *traffic_signals = i;
+            ++traffic_signals;
+        }
+    }
 
     return n;
 }

src/extractor/extractor.cpp

@@ -44,6 +44,7 @@
 #include <chrono>
 #include <fstream>
 #include <iostream>
+#include <iterator>
 #include <memory>
 #include <numeric> //partial_sum
 #include <thread>
@@ -409,33 +410,29 @@ std::shared_ptr<RestrictionMap> Extractor::LoadRestrictionMap()
   \brief Load node based graph from .osrm file
   */
 std::shared_ptr<util::NodeBasedDynamicGraph>
-Extractor::LoadNodeBasedGraph(std::unordered_set<NodeID> &barrier_nodes,
+Extractor::LoadNodeBasedGraph(std::unordered_set<NodeID> &barriers,
-                              std::unordered_set<NodeID> &traffic_lights,
+                              std::unordered_set<NodeID> &traffic_signals,
                               std::vector<QueryNode> &internal_to_external_node_map)
 {
-    std::vector<NodeBasedEdge> edge_list;
+    boost::filesystem::ifstream stream(config.output_file_name, std::ios::binary);
 
-    boost::filesystem::ifstream input_stream(config.output_file_name,
+    if (!stream)
-                                             std::ios::in | std::ios::binary);
+    {
+        throw util::exception("Unable to open " + config.output_file_name +
+                              " trying to read the node based graph");
+    }
+
+    auto barriers_iter = inserter(barriers, end(barriers));
+    auto traffic_signals_iter = inserter(traffic_signals, end(traffic_signals));
 
-    std::vector<NodeID> barrier_list;
-    std::vector<NodeID> traffic_light_list;
     NodeID number_of_node_based_nodes = util::loadNodesFromFile(
-        input_stream, barrier_list, traffic_light_list, internal_to_external_node_map);
+        stream, barriers_iter, traffic_signals_iter, internal_to_external_node_map);
 
-    util::SimpleLogger().Write() << " - " << barrier_list.size() << " bollard nodes, "
+    util::SimpleLogger().Write() << " - " << barriers.size() << " bollard nodes, "
-                                 << traffic_light_list.size() << " traffic lights";
+                                 << traffic_signals.size() << " traffic lights";
 
-    // insert into unordered sets for fast lookup
+    std::vector<NodeBasedEdge> edge_list;
-    barrier_nodes.insert(barrier_list.begin(), barrier_list.end());
+    util::loadEdgesFromFile(stream, edge_list);
-    traffic_lights.insert(traffic_light_list.begin(), traffic_light_list.end());
-
-    barrier_list.clear();
-    barrier_list.shrink_to_fit();
-    traffic_light_list.clear();
-    traffic_light_list.shrink_to_fit();
-
-    util::loadEdgesFromFile(input_stream, edge_list);
 
     if (edge_list.empty())
     {

src/tools/components.cpp

@@ -9,6 +9,7 @@
 #include "util/typedefs.hpp"
 
 #include <boost/filesystem.hpp>
+#include <boost/function_output_iterator.hpp>
 
 #if defined(__APPLE__) || defined(_WIN32)
 #include <gdal.h>
@@ -53,24 +54,20 @@ std::size_t loadGraph(const char *path,
                       std::vector<extractor::QueryNode> &coordinate_list,
                       std::vector<TarjanEdge> &graph_edge_list)
 {
-    std::ifstream input_stream(path, std::ifstream::in | std::ifstream::binary);
+    std::ifstream stream(path, std::ifstream::binary);
-    if (!input_stream.is_open())
+    if (!stream)
     {
         throw util::exception("Cannot open osrm file");
     }
 
     // load graph data
     std::vector<extractor::NodeBasedEdge> edge_list;
-    std::vector<NodeID> traffic_light_node_list;
-    std::vector<NodeID> barrier_node_list;
 
-    auto number_of_nodes = util::loadNodesFromFile(
+    auto nop = boost::make_function_output_iterator([](auto) {});
-        input_stream, barrier_node_list, traffic_light_node_list, coordinate_list);
 
-    util::loadEdgesFromFile(input_stream, edge_list);
+    auto number_of_nodes = util::loadNodesFromFile(stream, nop, nop, coordinate_list);
 
-    traffic_light_node_list.clear();
+    util::loadEdgesFromFile(stream, edge_list);
-    traffic_light_node_list.shrink_to_fit();
 
     // Building an node-based graph
     for (const auto &input_edge : edge_list)