refactor graph_loader (#3333)

Refactors the GraphLoader class to use the safer FileReader wrapper for better error handling.

include/util/graph_loader.hpp

@@ -5,6 +5,7 @@
 #include "extractor/node_based_edge.hpp"
 #include "extractor/query_node.hpp"
 #include "extractor/restriction.hpp"
+#include "storage/io.hpp"
 #include "util/exception.hpp"
 #include "util/fingerprint.hpp"
 #include "util/simple_logger.hpp"
@@ -33,25 +34,14 @@ namespace util
  * The since the restrictions reference nodes using their external node id,
  * we need to renumber it to the new internal id.
 */
-inline unsigned loadRestrictionsFromFile(std::istream &input_stream,
+inline unsigned loadRestrictionsFromFile(storage::io::FileReader &file_reader,
                                          std::vector<extractor::TurnRestriction> &restriction_list)
 {
-    const FingerPrint fingerprint_valid = FingerPrint::GetValid();
+    unsigned number_of_usable_restrictions = file_reader.ReadElementCount32();
-    FingerPrint fingerprint_loaded;
-    unsigned number_of_usable_restrictions = 0;
-    input_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint));
-    if (!fingerprint_loaded.TestContractor(fingerprint_valid))
-    {
-        SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n"
-                                            "Reprocess to get rid of this warning.";
-    }
-
-    input_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned));
     restriction_list.resize(number_of_usable_restrictions);
     if (number_of_usable_restrictions > 0)
     {
-        input_stream.read((char *)restriction_list.data(),
+        file_reader.ReadInto(restriction_list.data(), number_of_usable_restrictions);
-                          number_of_usable_restrictions * sizeof(extractor::TurnRestriction));
     }
 
     return number_of_usable_restrictions;
@@ -64,33 +54,24 @@ inline unsigned loadRestrictionsFromFile(std::istream &input_stream,
  *  - nodes indexed by their internal (non-osm) id
  */
 template <typename BarrierOutIter, typename TrafficSignalsOutIter>
-NodeID loadNodesFromFile(std::istream &input_stream,
+NodeID loadNodesFromFile(storage::io::FileReader &file_reader,
                          BarrierOutIter barriers,
                          TrafficSignalsOutIter traffic_signals,
                          std::vector<extractor::QueryNode> &node_array)
 {
-    const FingerPrint fingerprint_valid = FingerPrint::GetValid();
+    NodeID number_of_nodes = file_reader.ReadElementCount32();
-    FingerPrint fingerprint_loaded;
+    SimpleLogger().Write() << "Importing number_of_nodes new = " << number_of_nodes << " nodes ";
-    input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(FingerPrint));
 
-    if (!fingerprint_loaded.TestContractor(fingerprint_valid))
+    node_array.resize(number_of_nodes);
-    {
-        SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n"
-                                            "Reprocess to get rid of this warning.";
-    }
-
-    NodeID n;
-    input_stream.read(reinterpret_cast<char *>(&n), sizeof(NodeID));
-    SimpleLogger().Write() << "Importing n = " << n << " nodes ";
-
-    node_array.reserve(n);
 
     extractor::ExternalMemoryNode current_node;
-    for (NodeID i = 0; i < n; ++i)
+    for (NodeID i = 0; i < number_of_nodes; ++i)
     {
-        input_stream.read(reinterpret_cast<char *>(&current_node),
+        file_reader.ReadInto(&current_node, 1);
-                          sizeof(extractor::ExternalMemoryNode));
+
-        node_array.emplace_back(current_node.lon, current_node.lat, current_node.node_id);
+        node_array[i].lon = current_node.lon;
+        node_array[i].lat = current_node.lat;
+        node_array[i].node_id = current_node.node_id;
 
         if (current_node.barrier)
         {
@@ -105,21 +86,22 @@ NodeID loadNodesFromFile(std::istream &input_stream,
         }
     }
 
-    return n;
+    return number_of_nodes;
 }
 
 /**
  * Reads a .osrm file and produces the edges.
  */
-inline NodeID loadEdgesFromFile(std::istream &input_stream,
+inline NodeID loadEdgesFromFile(storage::io::FileReader &file_reader,
                                 std::vector<extractor::NodeBasedEdge> &edge_list)
 {
-    EdgeID m;
+    EdgeID number_of_edges = file_reader.ReadElementCount32();
-    input_stream.read(reinterpret_cast<char *>(&m), sizeof(unsigned));
+    BOOST_ASSERT(sizeof(EdgeID) == sizeof(number_of_edges));
-    edge_list.resize(m);
-    SimpleLogger().Write() << " and " << m << " edges ";
 
-    input_stream.read((char *)edge_list.data(), m * sizeof(extractor::NodeBasedEdge));
+    edge_list.resize(number_of_edges);
+    SimpleLogger().Write() << " and " << number_of_edges << " edges ";
+
+    file_reader.ReadInto(edge_list.data(), number_of_edges);
 
     BOOST_ASSERT(edge_list.size() > 0);
 
@@ -149,7 +131,7 @@ inline NodeID loadEdgesFromFile(std::istream &input_stream,
 
     SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges";
 
-    return m;
+    return number_of_edges;
 }
 }
 }

src/extractor/extractor.cpp

@@ -9,6 +9,7 @@
 #include "extractor/scripting_environment.hpp"
 
 #include "extractor/raster_source.hpp"
+#include "storage/io.hpp"
 #include "util/graph_loader.hpp"
 #include "util/io.hpp"
 #include "util/name_table.hpp"
@@ -399,11 +400,11 @@ void Extractor::FindComponents(unsigned max_edge_id,
   */
 std::shared_ptr<RestrictionMap> Extractor::LoadRestrictionMap()
 {
-    boost::filesystem::ifstream input_stream(config.restriction_file_name,
+    storage::io::FileReader file_reader(config.restriction_file_name,
-                                             std::ios::in | std::ios::binary);
+                                        storage::io::FileReader::VerifyFingerprint);
-
     std::vector<TurnRestriction> restriction_list;
-    util::loadRestrictionsFromFile(input_stream, restriction_list);
+
+    util::loadRestrictionsFromFile(file_reader, restriction_list);
 
     util::SimpleLogger().Write() << " - " << restriction_list.size() << " restrictions.";
 
@@ -418,25 +419,20 @@ Extractor::LoadNodeBasedGraph(std::unordered_set<NodeID> &barriers,
                               std::unordered_set<NodeID> &traffic_signals,
                               std::vector<QueryNode> &internal_to_external_node_map)
 {
-    boost::filesystem::ifstream stream(config.output_file_name, std::ios::binary);
+    storage::io::FileReader file_reader(config.output_file_name,
-
+                                        storage::io::FileReader::VerifyFingerprint);
-    if (!stream)
-    {
-        throw util::exception("Unable to open " + config.output_file_name +
-                              " trying to read the node based graph");
-    }
 
     auto barriers_iter = inserter(barriers, end(barriers));
     auto traffic_signals_iter = inserter(traffic_signals, end(traffic_signals));
 
     NodeID number_of_node_based_nodes = util::loadNodesFromFile(
-        stream, barriers_iter, traffic_signals_iter, internal_to_external_node_map);
+        file_reader, barriers_iter, traffic_signals_iter, internal_to_external_node_map);
 
     util::SimpleLogger().Write() << " - " << barriers.size() << " bollard nodes, "
                                  << traffic_signals.size() << " traffic lights";
 
     std::vector<NodeBasedEdge> edge_list;
-    util::loadEdgesFromFile(stream, edge_list);
+    util::loadEdgesFromFile(file_reader, edge_list);
 
     if (edge_list.empty())
     {

src/tools/components.cpp

@@ -50,24 +50,20 @@ void deleteFileIfExists(const std::string &file_name)
     }
 }
 
-std::size_t loadGraph(const char *path,
+std::size_t loadGraph(const std::string &path,
                       std::vector<extractor::QueryNode> &coordinate_list,
                       std::vector<TarjanEdge> &graph_edge_list)
 {
-    std::ifstream stream(path, std::ifstream::binary);
+    storage::io::FileReader file_reader(path, storage::io::FileReader::VerifyFingerprint);
-    if (!stream)
-    {
-        throw util::exception("Cannot open osrm file");
-    }
 
     // load graph data
     std::vector<extractor::NodeBasedEdge> edge_list;
 
     auto nop = boost::make_function_output_iterator([](auto) {});
 
-    auto number_of_nodes = util::loadNodesFromFile(stream, nop, nop, coordinate_list);
+    auto number_of_nodes = util::loadNodesFromFile(file_reader, nop, nop, coordinate_list);
 
-    util::loadEdgesFromFile(stream, edge_list);
+    util::loadEdgesFromFile(file_reader, edge_list);
 
     // Building an node-based graph
     for (const auto &input_edge : edge_list)
@@ -111,7 +107,8 @@ int main(int argc, char *argv[])
     }
 
     std::vector<osrm::tools::TarjanEdge> graph_edge_list;
-    auto number_of_nodes = osrm::tools::loadGraph(argv[1], coordinate_list, graph_edge_list);
+    auto number_of_nodes =
+        osrm::tools::loadGraph(std::string(argv[1]), coordinate_list, graph_edge_list);
 
     tbb::parallel_sort(graph_edge_list.begin(), graph_edge_list.end());
     const auto graph = std::make_shared<osrm::tools::TarjanGraph>(number_of_nodes, graph_edge_list);