Refactor graph writing code in contractor

include/contractor/contractor.hpp

@@ -68,9 +68,9 @@ class Contractor
     void WriteCoreNodeMarker(std::vector<bool> &&is_core_node) const;
     void WriteNodeLevels(std::vector<float> &&node_levels) const;
     void ReadNodeLevels(std::vector<float> &contraction_order) const;
-    std::size_t
+    void
     WriteContractedGraph(unsigned number_of_edge_based_nodes,
-                         const util::DeallocatingVector<QueryEdge> &contracted_edge_list);
+                         util::DeallocatingVector<QueryEdge> contracted_edge_list);
     void FindComponents(unsigned max_edge_id,
                         const util::DeallocatingVector<extractor::EdgeBasedEdge> &edges,
                         std::vector<extractor::EdgeBasedNode> &nodes) const;

include/contractor/files.hpp

@@ -0,0 +1,47 @@
+#ifndef OSRM_CONTRACTOR_FILES_HPP
+#define OSRM_CONTRACTOR_FILES_HPP
+
+#include "contractor/query_graph.hpp"
+
+#include "util/serialization.hpp"
+
+#include "storage/io.hpp"
+
+namespace osrm
+{
+namespace contractor
+{
+namespace files
+{
+
+// reads .osrm.hsgr file
+template<storage::Ownership Ownership>
+inline void readGraph(const boost::filesystem::path &path,
+                      unsigned &checksum,
+                      detail::QueryGraph<Ownership> &graph)
+{
+    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
+    storage::io::FileReader reader{path, fingerprint};
+
+    reader.ReadInto(checksum);
+    util::serialization::read(reader, graph);
+}
+
+// writes .osrm.hsgr file
+template<storage::Ownership Ownership>
+inline void writeGraph(const boost::filesystem::path &path,
+                       unsigned checksum,
+                       const detail::QueryGraph<Ownership> &graph)
+{
+    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
+    storage::io::FileWriter writer{path, fingerprint};
+
+    writer.WriteOne(checksum);
+    util::serialization::write(writer, graph);
+}
+
+}
+}
+}
+
+#endif

include/contractor/query_graph.hpp

@@ -0,0 +1,27 @@
+#ifndef OSRM_CONTRACTOR_QUERY_GRAPH_HPP
+#define OSRM_CONTRACTOR_QUERY_GRAPH_HPP
+
+#include "contractor/query_edge.hpp"
+
+#include "util/typedefs.hpp"
+#include "util/static_graph.hpp"
+
+#include <tuple>
+
+namespace osrm
+{
+namespace contractor
+{
+
+namespace detail
+{
+template <storage::Ownership Ownership> using QueryGraph = util::StaticGraph<typename QueryEdge::EdgeData, Ownership>;
+}
+
+using QueryGraph = detail::QueryGraph<storage::Ownership::Container>;
+using QueryGraphView = detail::QueryGraph<storage::Ownership::View>;
+
+}
+}
+
+#endif // QUERYEDGE_HPP

include/engine/datafacade/contiguous_internalmem_datafacade.hpp

@@ -17,6 +17,8 @@
 #include "extractor/segment_data_container.hpp"
 #include "extractor/turn_data_container.hpp"
 
+#include "contractor/query_graph.hpp"
+
 #include "partition/cell_storage.hpp"
 #include "partition/multi_level_partition.hpp"
 
@@ -62,11 +64,11 @@ template <>
 class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::AlgorithmDataFacade<CH>
 {
   private:
-    using QueryGraph = util::StaticGraph<EdgeData, storage::Ownership::View>;
+    using QueryGraph = contractor::QueryGraphView;
     using GraphNode = QueryGraph::NodeArrayEntry;
     using GraphEdge = QueryGraph::EdgeArrayEntry;
 
-    std::unique_ptr<QueryGraph> m_query_graph;
+    QueryGraph m_query_graph;
 
     // allocator that keeps the allocation data
     std::shared_ptr<ContiguousBlockAllocator> allocator;
@@ -83,7 +85,7 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
             graph_nodes_ptr, data_layout.num_entries[storage::DataLayout::CH_GRAPH_NODE_LIST]);
         util::vector_view<GraphEdge> edge_list(
             graph_edges_ptr, data_layout.num_entries[storage::DataLayout::CH_GRAPH_EDGE_LIST]);
-        m_query_graph.reset(new QueryGraph(node_list, edge_list));
+        m_query_graph = QueryGraph(node_list, edge_list);
     }
 
   public:
@@ -100,53 +102,53 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
     }
 
     // search graph access
-    unsigned GetNumberOfNodes() const override final { return m_query_graph->GetNumberOfNodes(); }
+    unsigned GetNumberOfNodes() const override final { return m_query_graph.GetNumberOfNodes(); }
 
-    unsigned GetNumberOfEdges() const override final { return m_query_graph->GetNumberOfEdges(); }
+    unsigned GetNumberOfEdges() const override final { return m_query_graph.GetNumberOfEdges(); }
 
     unsigned GetOutDegree(const NodeID n) const override final
     {
-        return m_query_graph->GetOutDegree(n);
+        return m_query_graph.GetOutDegree(n);
     }
 
-    NodeID GetTarget(const EdgeID e) const override final { return m_query_graph->GetTarget(e); }
+    NodeID GetTarget(const EdgeID e) const override final { return m_query_graph.GetTarget(e); }
 
-    EdgeData &GetEdgeData(const EdgeID e) const override final
+    const EdgeData &GetEdgeData(const EdgeID e) const override final
     {
-        return m_query_graph->GetEdgeData(e);
+        return m_query_graph.GetEdgeData(e);
     }
 
-    EdgeID BeginEdges(const NodeID n) const override final { return m_query_graph->BeginEdges(n); }
+    EdgeID BeginEdges(const NodeID n) const override final { return m_query_graph.BeginEdges(n); }
 
-    EdgeID EndEdges(const NodeID n) const override final { return m_query_graph->EndEdges(n); }
+    EdgeID EndEdges(const NodeID n) const override final { return m_query_graph.EndEdges(n); }
 
     EdgeRange GetAdjacentEdgeRange(const NodeID node) const override final
     {
-        return m_query_graph->GetAdjacentEdgeRange(node);
+        return m_query_graph.GetAdjacentEdgeRange(node);
     }
 
     // searches for a specific edge
     EdgeID FindEdge(const NodeID from, const NodeID to) const override final
     {
-        return m_query_graph->FindEdge(from, to);
+        return m_query_graph.FindEdge(from, to);
     }
 
     EdgeID FindEdgeInEitherDirection(const NodeID from, const NodeID to) const override final
     {
-        return m_query_graph->FindEdgeInEitherDirection(from, to);
+        return m_query_graph.FindEdgeInEitherDirection(from, to);
     }
 
     EdgeID
     FindEdgeIndicateIfReverse(const NodeID from, const NodeID to, bool &result) const override final
     {
-        return m_query_graph->FindEdgeIndicateIfReverse(from, to, result);
+        return m_query_graph.FindEdgeIndicateIfReverse(from, to, result);
     }
 
     EdgeID FindSmallestEdge(const NodeID from,
                             const NodeID to,
                             std::function<bool(EdgeData)> filter) const override final
     {
-        return m_query_graph->FindSmallestEdge(from, to, filter);
+        return m_query_graph.FindSmallestEdge(from, to, filter);
     }
 };
 

include/extractor/files.hpp

@@ -95,10 +95,10 @@ inline void writeTurnData(const boost::filesystem::path &path,
 }
 
 // reads .osrm.tls
-template <bool UseShareMemory>
+template <storage::Ownership Ownership>
 inline void readTurnLaneDescriptions(const boost::filesystem::path &path,
-                         typename util::ShM<std::uint32_t, UseShareMemory>::vector &turn_offsets,
+                         typename util::ShM<std::uint32_t, Ownership>::vector &turn_offsets,
-                         typename util::ShM<extractor::guidance::TurnLaneType::Mask, UseShareMemory>::vector &turn_masks)
+                         typename util::ShM<extractor::guidance::TurnLaneType::Mask, Ownership>::vector &turn_masks)
 {
     const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
     storage::io::FileReader reader{path, fingerprint};
@@ -108,10 +108,10 @@ inline void readTurnLaneDescriptions(const boost::filesystem::path &path,
 }
 
 // writes .osrm.tls
-template <bool UseShareMemory>
+template <storage::Ownership Ownership>
 inline void writeTurnLaneDescriptions(const boost::filesystem::path &path,
-                         const typename util::ShM<std::uint32_t, UseShareMemory>::vector &turn_offsets,
+                         const typename util::ShM<std::uint32_t, Ownership>::vector &turn_offsets,
-                         const typename util::ShM<extractor::guidance::TurnLaneType::Mask, UseShareMemory>::vector &turn_masks)
+                         const typename util::ShM<extractor::guidance::TurnLaneType::Mask, Ownership>::vector &turn_masks)
 {
     const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
     storage::io::FileWriter writer{path, fingerprint};

include/storage/serialization.hpp

@@ -29,47 +29,6 @@ namespace serialization
 // To make function calls consistent, this function returns the fixed number of properties
 inline std::size_t readPropertiesCount() { return 1; }
 
-struct HSGRHeader
-{
-    std::uint32_t checksum;
-    std::uint64_t number_of_nodes;
-    std::uint64_t number_of_edges;
-};
-
-// Reads the checksum, number of nodes and number of edges written in the header file of a `.hsgr`
-// file and returns them in a HSGRHeader struct
-inline HSGRHeader readHSGRHeader(io::FileReader &input_file)
-{
-
-    HSGRHeader header;
-    input_file.ReadInto(header.checksum);
-    input_file.ReadInto(header.number_of_nodes);
-    input_file.ReadInto(header.number_of_edges);
-
-    // If we have edges, then we must have nodes.
-    // However, there can be nodes with no edges (some test cases create this)
-    BOOST_ASSERT_MSG(header.number_of_edges == 0 || header.number_of_nodes > 0,
-                     "edges exist, but there are no nodes");
-
-    return header;
-}
-
-// Reads the graph data of a `.hsgr` file into memory
-// Needs to be called after readHSGRHeader() to get the correct offset in the stream
-using NodeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::NodeArrayEntry;
-using EdgeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::EdgeArrayEntry;
-inline void readHSGR(io::FileReader &input_file,
-                     NodeT *node_buffer,
-                     const std::uint64_t number_of_nodes,
-                     EdgeT *edge_buffer,
-                     const std::uint64_t number_of_edges)
-{
-    BOOST_ASSERT(node_buffer);
-    BOOST_ASSERT(edge_buffer);
-    input_file.ReadInto(node_buffer, number_of_nodes);
-    input_file.ReadInto(edge_buffer, number_of_edges);
-}
-
 // Loads coordinates and OSM node IDs from .nodes files into memory
 // Needs to be called after readElementCount() to get the correct offset in the stream
 template <typename OSMNodeIDVectorT>

include/util/deallocating_vector.hpp

@@ -239,6 +239,19 @@ class DeallocatingVector
         bucket_list.emplace_back(new ElementT[ELEMENTS_PER_BLOCK]);
     }
 
+    DeallocatingVector(DeallocatingVector &&other)
+    {
+        bucket_list = std::move(other.bucket_list);
+        current_size = std::move(other.current_size);
+    }
+
+    DeallocatingVector& operator=(DeallocatingVector &&other)
+    {
+        bucket_list = std::move(other.bucket_list);
+        current_size = std::move(other.current_size);
+        return *this;
+    }
+
     ~DeallocatingVector() { clear(); }
 
     friend void swap<>(DeallocatingVector<ElementT, ELEMENTS_PER_BLOCK> &lhs,

include/util/serialization.hpp

@@ -9,6 +9,8 @@ namespace osrm
 {
 namespace util
 {
+namespace serialization
+{
 
 template <typename EdgeDataT, storage::Ownership Ownership>
 inline void read(storage::io::FileReader &reader,
@@ -20,7 +22,7 @@ inline void read(storage::io::FileReader &reader,
 
 template <typename EdgeDataT, storage::Ownership Ownership>
 inline void write(storage::io::FileWriter &writer,
-                 StaticGraph<EdgeDataT, Ownership> &graph)
+                 const StaticGraph<EdgeDataT, Ownership> &graph)
 {
     writer.SerializeVector(graph.node_array);
     writer.SerializeVector(graph.edge_array);
@@ -43,7 +45,7 @@ inline void read(storage::io::FileReader &reader,
 
 template <typename EdgeDataT>
 inline void write(storage::io::FileWriter &writer,
-                 DynamicGraph<EdgeDataT> &graph)
+                 const DynamicGraph<EdgeDataT> &graph)
 {
     writer.SerializeVector(graph.node_array);
     writer.WriteElementCount64(graph.number_of_edges);
@@ -53,6 +55,7 @@ inline void write(storage::io::FileWriter &writer,
     }
 }
 
+}
 }
 }
 

include/util/static_graph.hpp

@@ -139,7 +139,7 @@ class StaticGraph
 
     StaticGraph() {}
 
-    template <typename ContainerT> StaticGraph(const int nodes, const ContainerT &edges)
+    template <typename ContainerT> StaticGraph(const std::uint32_t nodes, const ContainerT &edges)
     {
         BOOST_ASSERT(std::is_sorted(const_cast<ContainerT &>(edges).begin(),
                                     const_cast<ContainerT &>(edges).end()));
@@ -258,7 +258,7 @@ class StaticGraph
 
   protected:
     template <typename IterT>
-    void InitializeFromSortedEdgeRange(const unsigned nodes, IterT begin, IterT end)
+    void InitializeFromSortedEdgeRange(const std::uint32_t nodes, IterT begin, IterT end)
     {
         number_of_nodes = nodes;
         number_of_edges = static_cast<EdgeIterator>(std::distance(begin, end));
@@ -272,7 +272,7 @@ class StaticGraph
             unsigned offset = std::distance(begin, iter);
             node_array.push_back(NodeArrayEntry{offset});
         }
-        BOOST_ASSERT(iter == end);
+        BOOST_ASSERT_MSG(iter == end, ("Still " + std::to_string(std::distance(iter, end)) + " edges left.").c_str());
         BOOST_ASSERT(node_array.size() == number_of_nodes + 1);
 
         edge_array.resize(number_of_edges);

src/contractor/contractor.cpp

@@ -1,4 +1,5 @@
 #include "contractor/contractor.hpp"
+#include "contractor/files.hpp"
 #include "contractor/crc32_processor.hpp"
 #include "contractor/graph_contractor.hpp"
 #include "contractor/graph_contractor_adaptors.hpp"
@@ -84,7 +85,7 @@ int Contractor::Run()
 
     util::Log() << "Contraction took " << TIMER_SEC(contraction) << " sec";
 
-    std::size_t number_of_used_edges = WriteContractedGraph(max_edge_id, contracted_edge_list);
+    WriteContractedGraph(max_edge_id, std::move(contracted_edge_list));
     WriteCoreNodeMarker(std::move(is_core_node));
     if (!config.use_cached_priority)
     {
@@ -93,14 +94,7 @@ int Contractor::Run()
 
     TIMER_STOP(preparing);
 
-    const auto nodes_per_second =
-        static_cast<std::uint64_t>((max_edge_id + 1) / TIMER_SEC(contraction));
-    const auto edges_per_second =
-        static_cast<std::uint64_t>(number_of_used_edges / TIMER_SEC(contraction));
-
     util::Log() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds";
-    util::Log() << "Contraction: " << nodes_per_second << " nodes/sec and " << edges_per_second
-                << " edges/sec";
 
     util::Log() << "finished preprocessing";
 
@@ -144,117 +138,21 @@ void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const
     core_marker_output_file.WriteFrom(unpacked_bool_flags.data(), count);
 }
 
-std::size_t
+void
 Contractor::WriteContractedGraph(unsigned max_node_id,
-                                 const util::DeallocatingVector<QueryEdge> &contracted_edge_list)
+                                 util::DeallocatingVector<QueryEdge> contracted_edge_list)
 {
     // Sorting contracted edges in a way that the static query graph can read some in in-place.
     tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end());
-    const std::uint64_t contracted_edge_count = contracted_edge_list.size();
+    auto new_end = std::unique(contracted_edge_list.begin(), contracted_edge_list.end());
-    util::Log() << "Serializing compacted graph of " << contracted_edge_count << " edges";
+    contracted_edge_list.resize(new_end - contracted_edge_list.begin());
-
-    storage::io::FileWriter hsgr_output_file(config.graph_output_path,
-                                             storage::io::FileWriter::GenerateFingerprint);
-
-    const NodeID max_used_node_id = [&contracted_edge_list] {
-        NodeID tmp_max = 0;
-        for (const QueryEdge &edge : contracted_edge_list)
-        {
-            BOOST_ASSERT(SPECIAL_NODEID != edge.source);
-            BOOST_ASSERT(SPECIAL_NODEID != edge.target);
-            tmp_max = std::max(tmp_max, edge.source);
-            tmp_max = std::max(tmp_max, edge.target);
-        }
-        return tmp_max;
-    }();
-
-    util::Log(logDEBUG) << "input graph has " << (max_node_id + 1) << " nodes";
-    util::Log(logDEBUG) << "contracted graph has " << (max_used_node_id + 1) << " nodes";
-
-    std::vector<util::StaticGraph<EdgeData>::NodeArrayEntry> node_array;
-    // make sure we have at least one sentinel
-    node_array.resize(max_node_id + 2);
-
-    util::Log() << "Building node array";
-    util::StaticGraph<EdgeData>::EdgeIterator edge = 0;
-    util::StaticGraph<EdgeData>::EdgeIterator position = 0;
-    util::StaticGraph<EdgeData>::EdgeIterator last_edge;
-
-    // initializing 'first_edge'-field of nodes:
-    for (const auto node : util::irange(0u, max_used_node_id + 1))
-    {
-        last_edge = edge;
-        while ((edge < contracted_edge_count) && (contracted_edge_list[edge].source == node))
-        {
-            ++edge;
-        }
-        node_array[node].first_edge = position; //=edge
-        position += edge - last_edge;           // remove
-    }
-
-    for (const auto sentinel_counter :
-         util::irange<unsigned>(max_used_node_id + 1, node_array.size()))
-    {
-        // sentinel element, guarded against underflow
-        node_array[sentinel_counter].first_edge = contracted_edge_count;
-    }
-
-    util::Log() << "Serializing node array";
 
     RangebasedCRC32 crc32_calculator;
-    const unsigned edges_crc32 = crc32_calculator(contracted_edge_list);
+    const unsigned checksum = crc32_calculator(contracted_edge_list);
-    util::Log() << "Writing CRC32: " << edges_crc32;
 
-    const std::uint64_t node_array_size = node_array.size();
+    QueryGraph query_graph {max_node_id+1, contracted_edge_list};
-    // serialize crc32, aka checksum
-    hsgr_output_file.WriteOne(edges_crc32);
-    // serialize number of nodes
-    hsgr_output_file.WriteOne(node_array_size);
-    // serialize number of edges
-    hsgr_output_file.WriteOne(contracted_edge_count);
-    // serialize all nodes
-    if (node_array_size > 0)
-    {
-        hsgr_output_file.WriteFrom(node_array.data(), node_array_size);
-    }
 
-    // serialize all edges
+    files::writeGraph(config.graph_output_path, checksum, query_graph);
-    util::Log() << "Building edge array";
-    std::size_t number_of_used_edges = 0;
-
-    util::StaticGraph<EdgeData>::EdgeArrayEntry current_edge;
-    for (const auto edge : util::irange<std::size_t>(0UL, contracted_edge_list.size()))
-    {
-        // some self-loops are required for oneway handling. Need to assertthat we only keep these
-        // (TODO)
-        // no eigen loops
-        // BOOST_ASSERT(contracted_edge_list[edge].source != contracted_edge_list[edge].target ||
-        // node_represents_oneway[contracted_edge_list[edge].source]);
-        current_edge.target = contracted_edge_list[edge].target;
-        current_edge.data = contracted_edge_list[edge].data;
-
-        // every target needs to be valid
-        BOOST_ASSERT(current_edge.target <= max_used_node_id);
-#ifndef NDEBUG
-        if (current_edge.data.weight <= 0)
-        {
-            util::Log(logWARNING) << "Edge: " << edge
-                                  << ",source: " << contracted_edge_list[edge].source
-                                  << ", target: " << contracted_edge_list[edge].target
-                                  << ", weight: " << current_edge.data.weight;
-
-            util::Log(logWARNING) << "Failed at adjacency list of node "
-                                  << contracted_edge_list[edge].source << "/"
-                                  << node_array.size() - 1;
-            throw util::exception("Edge weight is <= 0" + SOURCE_REF);
-        }
-#endif
-        hsgr_output_file.WriteOne(current_edge);
-
-        ++number_of_used_edges;
-    }
-
-    return number_of_used_edges;
 }
 
 } // namespace contractor

src/storage/storage.cpp

@@ -1,6 +1,17 @@
 #include "storage/storage.hpp"
-#include "contractor/query_edge.hpp"
+
+#include "storage/io.hpp"
+#include "storage/serialization.hpp"
+#include "storage/shared_datatype.hpp"
+#include "storage/shared_memory.hpp"
+#include "storage/shared_memory_ownership.hpp"
+#include "storage/shared_monitor.hpp"
+
+#include "contractor/query_graph.hpp"
+#include "contractor/files.hpp"
+
 #include "customizer/edge_based_graph.hpp"
+
 #include "extractor/compressed_edge_container.hpp"
 #include "extractor/edge_based_edge.hpp"
 #include "extractor/files.hpp"
@@ -9,17 +20,14 @@
 #include "extractor/profile_properties.hpp"
 #include "extractor/query_node.hpp"
 #include "extractor/travel_mode.hpp"
+
 #include "partition/cell_storage.hpp"
 #include "partition/edge_based_graph_reader.hpp"
 #include "partition/files.hpp"
 #include "partition/multi_level_partition.hpp"
-#include "storage/io.hpp"
+
-#include "storage/serialization.hpp"
-#include "storage/shared_datatype.hpp"
-#include "storage/shared_memory.hpp"
-#include "storage/shared_memory_ownership.hpp"
-#include "storage/shared_monitor.hpp"
 #include "engine/datafacade/datafacade_base.hpp"
+
 #include "util/coordinate.hpp"
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
@@ -56,8 +64,7 @@ namespace storage
 {
 
 using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf;
-using RTreeNode = util::
+using RTreeNode = util:: StaticRTree<RTreeLeaf, util::vector_view<util::Coordinate>, storage::Ownership::View>::TreeNode;
-    StaticRTree<RTreeLeaf, util::vector_view<util::Coordinate>, storage::Ownership::View>::TreeNode;
 using QueryGraph = util::StaticGraph<contractor::QueryEdge::EdgeData>;
 using EdgeBasedGraph = util::StaticGraph<extractor::EdgeBasedEdge::EdgeData>;
 
@@ -244,20 +251,23 @@ void Storage::PopulateLayout(DataLayout &layout)
 
     if (boost::filesystem::exists(config.hsgr_data_path))
     {
-        io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::VerifyFingerprint);
+        io::FileReader reader(config.hsgr_data_path, io::FileReader::VerifyFingerprint);
+
+        reader.Skip<std::uint32_t>(1); // checksum
+        auto num_nodes = reader.ReadVectorSize<contractor::QueryGraph::NodeArrayEntry>();
+        auto num_edges = reader.ReadVectorSize<contractor::QueryGraph::EdgeArrayEntry>();
 
-        const auto hsgr_header = serialization::readHSGRHeader(hsgr_file);
         layout.SetBlockSize<unsigned>(DataLayout::HSGR_CHECKSUM, 1);
-        layout.SetBlockSize<QueryGraph::NodeArrayEntry>(DataLayout::CH_GRAPH_NODE_LIST,
+        layout.SetBlockSize<contractor::QueryGraph::NodeArrayEntry>(DataLayout::CH_GRAPH_NODE_LIST,
-                                                        hsgr_header.number_of_nodes);
+                                                        num_nodes);
-        layout.SetBlockSize<QueryGraph::EdgeArrayEntry>(DataLayout::CH_GRAPH_EDGE_LIST,
+        layout.SetBlockSize<contractor::QueryGraph::EdgeArrayEntry>(DataLayout::CH_GRAPH_EDGE_LIST,
-                                                        hsgr_header.number_of_edges);
+                                                        num_edges);
     }
     else
     {
         layout.SetBlockSize<unsigned>(DataLayout::HSGR_CHECKSUM, 0);
-        layout.SetBlockSize<QueryGraph::NodeArrayEntry>(DataLayout::CH_GRAPH_NODE_LIST, 0);
+        layout.SetBlockSize<contractor::QueryGraph::NodeArrayEntry>(DataLayout::CH_GRAPH_NODE_LIST, 0);
-        layout.SetBlockSize<QueryGraph::EdgeArrayEntry>(DataLayout::CH_GRAPH_EDGE_LIST, 0);
+        layout.SetBlockSize<contractor::QueryGraph::EdgeArrayEntry>(DataLayout::CH_GRAPH_EDGE_LIST, 0);
     }
 
     // load rsearch tree size
@@ -480,34 +490,26 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // Load the HSGR file
     if (boost::filesystem::exists(config.hsgr_data_path))
     {
-        io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::VerifyFingerprint);
+        auto graph_nodes_ptr = layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
-        auto hsgr_header = serialization::readHSGRHeader(hsgr_file);
+            memory_ptr, storage::DataLayout::CH_GRAPH_NODE_LIST);
-        unsigned *checksum_ptr =
+        auto graph_edges_ptr = layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
-            layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
+            memory_ptr, storage::DataLayout::CH_GRAPH_EDGE_LIST);
-        *checksum_ptr = hsgr_header.checksum;
+        auto checksum = layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
 
-        // load the nodes of the search graph
+        util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
-        QueryGraph::NodeArrayEntry *graph_node_list_ptr =
+            graph_nodes_ptr, layout.num_entries[storage::DataLayout::CH_GRAPH_NODE_LIST]);
-            layout.GetBlockPtr<QueryGraph::NodeArrayEntry, true>(memory_ptr,
+        util::vector_view<contractor::QueryGraphView::EdgeArrayEntry> edge_list(
-                                                                 DataLayout::CH_GRAPH_NODE_LIST);
+            graph_edges_ptr, layout.num_entries[storage::DataLayout::CH_GRAPH_EDGE_LIST]);
 
-        // load the edges of the search graph
+        contractor::QueryGraphView graph_view(std::move(node_list), std::move(edge_list));
-        QueryGraph::EdgeArrayEntry *graph_edge_list_ptr =
+        contractor::files::readGraph(config.hsgr_data_path, *checksum, graph_view);
-            layout.GetBlockPtr<QueryGraph::EdgeArrayEntry, true>(memory_ptr,
-                                                                 DataLayout::CH_GRAPH_EDGE_LIST);
-
-        serialization::readHSGR(hsgr_file,
-                                graph_node_list_ptr,
-                                hsgr_header.number_of_nodes,
-                                graph_edge_list_ptr,
-                                hsgr_header.number_of_edges);
     }
     else
     {
         layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
-        layout.GetBlockPtr<QueryGraph::NodeArrayEntry, true>(memory_ptr,
+        layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(memory_ptr,
                                                              DataLayout::CH_GRAPH_NODE_LIST);
-        layout.GetBlockPtr<QueryGraph::EdgeArrayEntry, true>(memory_ptr,
+        layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(memory_ptr,
                                                              DataLayout::CH_GRAPH_EDGE_LIST);
     }
 
@@ -558,7 +560,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     {
         auto offsets_ptr = layout.GetBlockPtr<std::uint32_t, true>(
             memory_ptr, storage::DataLayout::LANE_DESCRIPTION_OFFSETS);
-        util::ShM<std::uint32_t, true>::vector offsets(
+        util::vector_view<std::uint32_t> offsets(
             offsets_ptr, layout.num_entries[storage::DataLayout::LANE_DESCRIPTION_OFFSETS]);
 
         auto masks_ptr = layout.GetBlockPtr<extractor::guidance::TurnLaneType::Mask, true>(
@@ -566,7 +568,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
         util::vector_view<extractor::guidance::TurnLaneType::Mask> masks(
             masks_ptr, layout.num_entries[storage::DataLayout::LANE_DESCRIPTION_MASKS]);
 
-        extractor::files::readTurnLaneDescriptions<true>(config.turn_lane_description_path, offsets, masks);
+        extractor::files::readTurnLaneDescriptions<storage::Ownership::View>(config.turn_lane_description_path, offsets, masks);
     }
 
     // Load original edge data