Refactored loading code for .hsgr file

include/engine/datafacade/internal_datafacade.hpp

@@ -14,6 +14,7 @@
 #include "extractor/profile_properties.hpp"
 #include "extractor/query_node.hpp"
 #include "storage/storage_config.hpp"
+#include "storage/io.hpp"
 #include "engine/geospatial_query.hpp"
 #include "util/graph_loader.hpp"
 #include "util/guidance/turn_lanes.hpp"
@@ -150,21 +151,26 @@ class InternalDataFacade final : public BaseDataFacade
 
     void LoadGraph(const boost::filesystem::path &hsgr_path)
     {
-        util::ShM<QueryGraph::NodeArrayEntry, false>::vector node_list;
+        boost::filesystem::ifstream hsgr_input_stream(hsgr_path);
-        util::ShM<QueryGraph::EdgeArrayEntry, false>::vector edge_list;
+        if (!hsgr_input_stream)
+        {
+            throw util::exception("Could not open " + hsgr_path.string() + " for reading.");
+        }
 
-        util::SimpleLogger().Write() << "loading graph from " << hsgr_path.string();
+        auto header = storage::io::readHSGRHeader(hsgr_input_stream);
+        m_check_sum = header.checksum;
 
-        m_number_of_nodes = readHSGRFromStream(hsgr_path, node_list, edge_list, &m_check_sum);
+        util::ShM<QueryGraph::NodeArrayEntry, false>::vector node_list(header.number_of_nodes);
+        util::ShM<QueryGraph::EdgeArrayEntry, false>::vector edge_list(header.number_of_edges);
+
+        storage::io::readHSGR(hsgr_input_stream,
+                              node_list.data(),
+                              header.number_of_nodes,
+                              edge_list.data(),
+                              header.number_of_edges);
 
-        BOOST_ASSERT_MSG(0 != node_list.size(), "node list empty");
-        // BOOST_ASSERT_MSG(0 != edge_list.size(), "edge list empty");
-        util::SimpleLogger().Write() << "loaded " << node_list.size() << " nodes and "
-                                     << edge_list.size() << " edges";
         m_query_graph = std::unique_ptr<QueryGraph>(new QueryGraph(node_list, edge_list));
 
-        BOOST_ASSERT_MSG(0 == node_list.size(), "node list not flushed");
-        BOOST_ASSERT_MSG(0 == edge_list.size(), "edge list not flushed");
         util::SimpleLogger().Write() << "Data checksum is " << m_check_sum;
     }
 

include/storage/io.hpp

@@ -0,0 +1,68 @@
+#ifndef OSRM_STORAGE_IO_HPP_
+#define OSRM_STORAGE_IO_HPP_
+
+#include "util/fingerprint.hpp"
+#include "util/simple_logger.hpp"
+
+#include <boost/filesystem/fstream.hpp>
+
+#include <tuple>
+
+namespace osrm
+{
+namespace storage
+{
+namespace io
+{
+
+#pragma pack(push, 1)
+struct HSGRHeader
+{
+    std::uint32_t checksum;
+    std::uint32_t number_of_nodes;
+    std::uint32_t number_of_edges;
+};
+#pragma pack(pop)
+static_assert(sizeof(HSGRHeader) == 12, "HSGRHeader is not packed");
+
+// Returns the checksum and the number of nodes and number of edges
+inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream)
+{
+    const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid();
+    util::FingerPrint fingerprint_loaded;
+    input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(util::FingerPrint));
+    if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid))
+    {
+        util::SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build.\n"
+                                                  "Reprocess to get rid of this warning.";
+    }
+
+    HSGRHeader header;
+    input_stream.read(reinterpret_cast<char *>(&header.checksum), sizeof(header.checksum));
+    input_stream.read(reinterpret_cast<char *>(&header.number_of_nodes), sizeof(header.number_of_nodes));
+    input_stream.read(reinterpret_cast<char *>(&header.number_of_edges), sizeof(header.number_of_edges));
+
+    BOOST_ASSERT_MSG(0 != header.number_of_nodes, "number of nodes is zero");
+    // number of edges can be zero, this is the case in a few test fixtures
+
+    return header;
+}
+
+// Needs to be called after getHSGRSize() to get the correct offset in the stream
+// 
+template <typename NodeT, typename EdgeT>
+void readHSGR(boost::filesystem::ifstream &input_stream,
+              NodeT *node_buffer,
+              std::uint32_t number_of_nodes,
+              EdgeT *edge_buffer,
+              std::uint32_t number_of_edges)
+{
+    input_stream.read(reinterpret_cast<char *>(node_buffer), number_of_nodes * sizeof(NodeT));
+    input_stream.read(reinterpret_cast<char *>(edge_buffer), number_of_edges * sizeof(EdgeT));
+}
+
+}
+}
+}
+
+#endif

include/util/fingerprint.hpp

@@ -32,6 +32,7 @@ class FingerPrint
     boost::uuids::uuid named_uuid;
 };
 
+static_assert(sizeof(FingerPrint) == 152, "FingerPrint has unexpected size");
 static_assert(std::is_trivial<FingerPrint>::value, "FingerPrint needs to be trivial.");
 }
 }

include/util/graph_loader.hpp

@@ -148,56 +148,6 @@ NodeID loadEdgesFromFile(std::istream &input_stream,
     return m;
 }
 
-template <typename NodeT, typename EdgeT>
-unsigned readHSGRFromStream(const boost::filesystem::path &hsgr_file,
-                            std::vector<NodeT> &node_list,
-                            std::vector<EdgeT> &edge_list,
-                            unsigned *check_sum)
-{
-    if (!boost::filesystem::exists(hsgr_file))
-    {
-        throw exception("hsgr file does not exist");
-    }
-    if (0 == boost::filesystem::file_size(hsgr_file))
-    {
-        throw exception("hsgr file is empty");
-    }
-
-    boost::filesystem::ifstream hsgr_input_stream(hsgr_file, std::ios::binary);
-
-    const FingerPrint fingerprint_valid = FingerPrint::GetValid();
-    FingerPrint fingerprint_loaded;
-    hsgr_input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(FingerPrint));
-    if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid))
-    {
-        SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build.\n"
-                                            "Reprocess to get rid of this warning.";
-    }
-
-    unsigned number_of_nodes = 0;
-    unsigned number_of_edges = 0;
-    hsgr_input_stream.read(reinterpret_cast<char *>(check_sum), sizeof(unsigned));
-    hsgr_input_stream.read(reinterpret_cast<char *>(&number_of_nodes), sizeof(unsigned));
-    BOOST_ASSERT_MSG(0 != number_of_nodes, "number of nodes is zero");
-    hsgr_input_stream.read(reinterpret_cast<char *>(&number_of_edges), sizeof(unsigned));
-
-    SimpleLogger().Write() << "number_of_nodes: " << number_of_nodes
-                           << ", number_of_edges: " << number_of_edges;
-
-    // BOOST_ASSERT_MSG( 0 != number_of_edges, "number of edges is zero");
-    node_list.resize(number_of_nodes);
-    hsgr_input_stream.read(reinterpret_cast<char *>(&node_list[0]),
-                           number_of_nodes * sizeof(NodeT));
-
-    edge_list.resize(number_of_edges);
-    if (number_of_edges > 0)
-    {
-        hsgr_input_stream.read(reinterpret_cast<char *>(&edge_list[0]),
-                               number_of_edges * sizeof(EdgeT));
-    }
-
-    return number_of_nodes;
-}
 }
 }
 

src/storage/storage.cpp

@@ -1,3 +1,10 @@
+#include "storage/storage.hpp"
+
+#include "storage/io.hpp"
+#include "storage/shared_barriers.hpp"
+#include "storage/shared_datatype.hpp"
+#include "storage/shared_memory.hpp"
+
 #include "contractor/query_edge.hpp"
 #include "extractor/compressed_edge_container.hpp"
 #include "extractor/guidance/turn_instruction.hpp"
@@ -5,10 +12,6 @@
 #include "extractor/profile_properties.hpp"
 #include "extractor/query_node.hpp"
 #include "extractor/travel_mode.hpp"
-#include "storage/shared_barriers.hpp"
-#include "storage/shared_datatype.hpp"
-#include "storage/shared_memory.hpp"
-#include "storage/storage.hpp"
 #include "engine/datafacade/datafacade_base.hpp"
 #include "util/coordinate.hpp"
 #include "util/exception.hpp"
@@ -252,37 +255,12 @@ Storage::ReturnCode Storage::Run(int max_wait)
         throw util::exception("Could not open " + config.hsgr_data_path.string() + " for reading.");
     }
 
-    util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid();
+    auto hsgr_header = io::readHSGRHeader(hsgr_input_stream);
-    util::FingerPrint fingerprint_loaded;
-    hsgr_input_stream.read((char *)&fingerprint_loaded, sizeof(util::FingerPrint));
-    if (fingerprint_loaded.TestGraphUtil(fingerprint_valid))
-    {
-        util::SimpleLogger().Write(logDEBUG) << "Fingerprint checked out ok";
-    }
-    else
-    {
-        util::SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build. "
-                                                  "Reprocess to get rid of this warning.";
-    }
-
-    // load checksum
-    unsigned checksum = 0;
-    hsgr_input_stream.read((char *)&checksum, sizeof(unsigned));
     shared_layout_ptr->SetBlockSize<unsigned>(SharedDataLayout::HSGR_CHECKSUM, 1);
-    // load graph node size
-    unsigned number_of_graph_nodes = 0;
-    hsgr_input_stream.read((char *)&number_of_graph_nodes, sizeof(unsigned));
-
-    BOOST_ASSERT_MSG((0 != number_of_graph_nodes), "number of nodes is zero");
     shared_layout_ptr->SetBlockSize<QueryGraph::NodeArrayEntry>(SharedDataLayout::GRAPH_NODE_LIST,
-                                                                number_of_graph_nodes);
+                                                                hsgr_header.number_of_nodes);
-
-    // load graph edge size
-    unsigned number_of_graph_edges = 0;
-    hsgr_input_stream.read((char *)&number_of_graph_edges, sizeof(unsigned));
-    // BOOST_ASSERT_MSG(0 != number_of_graph_edges, "number of graph edges is zero");
     shared_layout_ptr->SetBlockSize<QueryGraph::EdgeArrayEntry>(SharedDataLayout::GRAPH_EDGE_LIST,
-                                                                number_of_graph_edges);
+                                                                hsgr_header.number_of_edges);
 
     // load rsearch tree size
     boost::filesystem::ifstream tree_node_file(config.ram_index_path, std::ios::binary);
@@ -477,7 +455,7 @@ Storage::ReturnCode Storage::Run(int max_wait)
     // hsgr checksum
     unsigned *checksum_ptr = shared_layout_ptr->GetBlockPtr<unsigned, true>(
         shared_memory_ptr, SharedDataLayout::HSGR_CHECKSUM);
-    *checksum_ptr = checksum;
+    *checksum_ptr = hsgr_header.checksum;
 
     // ram index file name
     char *file_index_path_ptr = shared_layout_ptr->GetBlockPtr<char, true>(
@@ -726,21 +704,17 @@ Storage::ReturnCode Storage::Run(int max_wait)
     QueryGraph::NodeArrayEntry *graph_node_list_ptr =
         shared_layout_ptr->GetBlockPtr<QueryGraph::NodeArrayEntry, true>(
             shared_memory_ptr, SharedDataLayout::GRAPH_NODE_LIST);
-    if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST) > 0)
-    {
-        hsgr_input_stream.read((char *)graph_node_list_ptr,
-                               shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST));
-    }
 
     // load the edges of the search graph
     QueryGraph::EdgeArrayEntry *graph_edge_list_ptr =
         shared_layout_ptr->GetBlockPtr<QueryGraph::EdgeArrayEntry, true>(
             shared_memory_ptr, SharedDataLayout::GRAPH_EDGE_LIST);
-    if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST) > 0)
+
-    {
+    io::readHSGR(hsgr_input_stream,
-        hsgr_input_stream.read((char *)graph_edge_list_ptr,
+                 graph_node_list_ptr,
-                               shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST));
+                 hsgr_header.number_of_nodes,
-    }
+                 graph_edge_list_ptr,
+                 hsgr_header.number_of_edges);
     hsgr_input_stream.close();
 
     // load profile properties