Add edge-based graph loading in MLD facade

include/contractor/graph_contractor_adaptors.hpp

@@ -22,33 +22,33 @@ std::vector<ContractorEdge> adaptToContractorInput(InputEdgeContainer input_edge
     {
 #ifndef NDEBUG
         const unsigned int constexpr DAY_IN_DECI_SECONDS = 24 * 60 * 60 * 10;
-        if (static_cast<unsigned int>(std::max(input_edge.weight, 1)) > DAY_IN_DECI_SECONDS)
+        if (static_cast<unsigned int>(std::max(input_edge.data.weight, 1)) > DAY_IN_DECI_SECONDS)
         {
             util::Log(logWARNING) << "Edge weight large -> "
-                                  << static_cast<unsigned int>(std::max(input_edge.weight, 1))
+                                  << static_cast<unsigned int>(std::max(input_edge.data.weight, 1))
                                   << " : " << static_cast<unsigned int>(input_edge.source) << " -> "
                                   << static_cast<unsigned int>(input_edge.target);
         }
 #endif
         edges.emplace_back(input_edge.source,
                            input_edge.target,
-                           std::max(input_edge.weight, 1),
+                           std::max(input_edge.data.weight, 1),
-                           input_edge.duration,
+                           input_edge.data.duration,
                            1,
-                           input_edge.edge_id,
+                           input_edge.data.edge_id,
                            false,
-                           input_edge.forward ? true : false,
+                           input_edge.data.forward ? true : false,
-                           input_edge.backward ? true : false);
+                           input_edge.data.backward ? true : false);
 
         edges.emplace_back(input_edge.target,
                            input_edge.source,
-                           std::max(input_edge.weight, 1),
+                           std::max(input_edge.data.weight, 1),
-                           input_edge.duration,
+                           input_edge.data.duration,
                            1,
-                           input_edge.edge_id,
+                           input_edge.data.edge_id,
                            false,
-                           input_edge.backward ? true : false,
+                           input_edge.data.backward ? true : false,
-                           input_edge.forward ? true : false);
+                           input_edge.data.forward ? true : false);
     }
     // FIXME not sure if we need this
     edges.shrink_to_fit();

include/engine/datafacade/algorithm_datafacade.hpp

@@ -2,6 +2,7 @@
 #define OSRM_ENGINE_DATAFACADE_ALGORITHM_DATAFACADE_HPP
 
 #include "contractor/query_edge.hpp"
+#include "extractor/edge_based_edge.hpp"
 #include "engine/algorithm.hpp"
 
 #include "partition/cell_storage.hpp"
@@ -66,6 +67,25 @@ template <> class AlgorithmDataFacade<algorithm::CoreCH>
 template <> class AlgorithmDataFacade<algorithm::MLD>
 {
   public:
+    using EdgeData = extractor::EdgeBasedEdge::EdgeData;
+
+    // search graph access
+    virtual unsigned GetNumberOfNodes() const = 0;
+
+    virtual unsigned GetNumberOfEdges() const = 0;
+
+    virtual unsigned GetOutDegree(const NodeID n) const = 0;
+
+    virtual NodeID GetTarget(const EdgeID e) const = 0;
+
+    virtual const EdgeData &GetEdgeData(const EdgeID e) const = 0;
+
+    virtual EdgeID BeginEdges(const NodeID n) const = 0;
+
+    virtual EdgeID EndEdges(const NodeID n) const = 0;
+
+    virtual EdgeRange GetAdjacentEdgeRange(const NodeID node) const = 0;
+
     virtual const partition::MultiLevelPartitionView &GetMultiLevelPartition() const = 0;
 
     virtual const partition::CellStorageView &GetCellStorage() const = 0;

include/engine/datafacade/contiguous_internalmem_datafacade.hpp

@@ -1200,12 +1200,67 @@ class ContiguousInternalMemoryDataFacade<algorithm::MLD>
     : public ContiguousInternalMemoryDataFacadeBase,
       public ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>
 {
+  private:
+    using QueryGraph = util::StaticGraph<EdgeData, true>;
+    using GraphNode = QueryGraph::NodeArrayEntry;
+    using GraphEdge = QueryGraph::EdgeArrayEntry;
+
+    std::unique_ptr<QueryGraph> m_query_graph;
+
+    void InitializeGraphPointer(storage::DataLayout &data_layout, char *memory_block)
+    {
+        auto graph_nodes_ptr = data_layout.GetBlockPtr<GraphNode>(
+            memory_block, storage::DataLayout::MLD_GRAPH_NODE_LIST);
+
+        auto graph_edges_ptr = data_layout.GetBlockPtr<GraphEdge>(
+            memory_block, storage::DataLayout::MLD_GRAPH_EDGE_LIST);
+
+        util::ShM<GraphNode, true>::vector node_list(
+            graph_nodes_ptr, data_layout.num_entries[storage::DataLayout::MLD_GRAPH_NODE_LIST]);
+        util::ShM<GraphEdge, true>::vector edge_list(
+            graph_edges_ptr, data_layout.num_entries[storage::DataLayout::MLD_GRAPH_EDGE_LIST]);
+
+        m_query_graph.reset(new QueryGraph(node_list, edge_list));
+    }
+
   public:
     ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator)
         : ContiguousInternalMemoryDataFacadeBase(allocator),
           ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>(allocator)
 
     {
+        InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory());
+    }
+
+    void InitializeInternalPointers(storage::DataLayout &data_layout, char *memory_block)
+    {
+        InitializeGraphPointer(data_layout, memory_block);
+    }
+
+    // search graph access
+    unsigned GetNumberOfNodes() const override final { return m_query_graph->GetNumberOfNodes(); }
+
+    unsigned GetNumberOfEdges() const override final { return m_query_graph->GetNumberOfEdges(); }
+
+    unsigned GetOutDegree(const NodeID n) const override final
+    {
+        return m_query_graph->GetOutDegree(n);
+    }
+
+    NodeID GetTarget(const EdgeID e) const override final { return m_query_graph->GetTarget(e); }
+
+    EdgeData &GetEdgeData(const EdgeID e) const override final
+    {
+        return m_query_graph->GetEdgeData(e);
+    }
+
+    EdgeID BeginEdges(const NodeID n) const override final { return m_query_graph->BeginEdges(n); }
+
+    EdgeID EndEdges(const NodeID n) const override final { return m_query_graph->EndEdges(n); }
+
+    EdgeRange GetAdjacentEdgeRange(const NodeID node) const override final
+    {
+        return m_query_graph->GetAdjacentEdgeRange(node);
     }
 };
 }

include/extractor/edge_based_edge.hpp

@@ -29,11 +29,29 @@ struct EdgeBasedEdge
 
     NodeID source;
     NodeID target;
-    NodeID edge_id;
+
-    EdgeWeight weight;
+    struct EdgeData
-    EdgeWeight duration : 30;
+    {
-    std::uint32_t forward : 1;
+        EdgeData() : edge_id(0), weight(0), duration(0), forward(false), backward(false) {}
-    std::uint32_t backward : 1;
+
+        EdgeData(const NodeID edge_id,
+                 const EdgeWeight weight,
+                 const EdgeWeight duration,
+                 const bool forward,
+                 const bool backward)
+            : edge_id(edge_id), weight(weight), duration(duration), forward(forward),
+              backward(backward)
+        {
+        }
+
+        NodeID edge_id;
+        EdgeWeight weight;
+        EdgeWeight duration : 30;
+        std::uint32_t forward : 1;
+        std::uint32_t backward : 1;
+
+        auto is_unidirectional() const { return !forward || !backward; }
+    } data;
 };
 static_assert(sizeof(extractor::EdgeBasedEdge) == 20,
               "Size of extractor::EdgeBasedEdge type is "
@@ -42,10 +60,7 @@ static_assert(sizeof(extractor::EdgeBasedEdge) == 20,
 
 // Impl.
 
-inline EdgeBasedEdge::EdgeBasedEdge()
+inline EdgeBasedEdge::EdgeBasedEdge() : source(0), target(0) {}
-    : source(0), target(0), edge_id(0), weight(0), duration(0), forward(false), backward(false)
-{
-}
 
 inline EdgeBasedEdge::EdgeBasedEdge(const NodeID source,
                                     const NodeID target,
@@ -54,19 +69,18 @@ inline EdgeBasedEdge::EdgeBasedEdge(const NodeID source,
                                     const EdgeWeight duration,
                                     const bool forward,
                                     const bool backward)
-    : source(source), target(target), edge_id(edge_id), weight(weight), duration(duration),
+    : source(source), target(target), data{edge_id, weight, duration, forward, backward}
-      forward(forward), backward(backward)
 {
 }
 
 inline bool EdgeBasedEdge::operator<(const EdgeBasedEdge &other) const
 {
-    const auto unidirectional = (!forward || !backward);
+    const auto unidirectional = data.is_unidirectional();
-    const auto other_is_unidirectional = (!other.forward || !other.backward);
+    const auto other_is_unidirectional = other.data.is_unidirectional();
     // if all items are the same, we want to keep bidirectional edges. due to the `<` operator,
     // preferring 0 (false) over 1 (true), we need to compare the inverse of `bidirectional`
-    return std::tie(source, target, weight, unidirectional) <
+    return std::tie(source, target, data.weight, unidirectional) <
-           std::tie(other.source, other.target, other.weight, other_is_unidirectional);
+           std::tie(other.source, other.target, other.data.weight, other_is_unidirectional);
 }
 } // ns extractor
 } // ns osrm

include/partition/edge_based_graph_reader.hpp

@@ -19,7 +19,7 @@ namespace osrm
 namespace partition
 {
 
-struct EdgeBasedGraphEdgeData : extractor::EdgeBasedEdge
+struct EdgeBasedGraphEdgeData : extractor::EdgeBasedEdge::EdgeData
 {
     // We need to write out the full edge based graph again.
 
@@ -38,6 +38,107 @@ struct EdgeBasedGraphEdge : EdgeBasedGraph::InputEdge
     using Base::Base;
 };
 
+// Bidirectional (s,t) to (s,t) and (t,s)
+std::vector<extractor::EdgeBasedEdge>
+SplitBidirectionalEdges(const std::vector<extractor::EdgeBasedEdge> &edges)
+{
+    std::vector<extractor::EdgeBasedEdge> directed;
+    directed.reserve(edges.size() * 2);
+
+    for (const auto &edge : edges)
+    {
+        directed.emplace_back(edge.source,
+                              edge.target,
+                              edge.data.edge_id,
+                              std::max(edge.data.weight, 1),
+                              edge.data.duration,
+                              edge.data.forward,
+                              edge.data.backward);
+
+        directed.emplace_back(edge.target,
+                              edge.source,
+                              edge.data.edge_id,
+                              std::max(edge.data.weight, 1),
+                              edge.data.duration,
+                              edge.data.backward,
+                              edge.data.forward);
+    }
+
+    return directed;
+}
+
+std::vector<EdgeBasedGraphEdge>
+PrepareEdgesForUsageInGraph(std::vector<extractor::EdgeBasedEdge> edges)
+{
+    std::sort(begin(edges), end(edges));
+
+    std::vector<EdgeBasedGraphEdge> graph_edges;
+    graph_edges.reserve(edges.size());
+
+    for (NodeID i = 0; i < edges.size();)
+    {
+        const NodeID source = edges[i].source;
+        const NodeID target = edges[i].target;
+
+        // remove eigenloops
+        if (source == target)
+        {
+            ++i;
+            continue;
+        }
+
+        EdgeBasedGraphEdge forward_edge;
+        EdgeBasedGraphEdge reverse_edge;
+        forward_edge.source = reverse_edge.source = source;
+        forward_edge.target = reverse_edge.target = target;
+        forward_edge.data.edge_id = reverse_edge.data.edge_id = edges[i].data.edge_id;
+        forward_edge.data.weight = reverse_edge.data.weight = INVALID_EDGE_WEIGHT;
+        forward_edge.data.duration = reverse_edge.data.duration = MAXIMAL_EDGE_DURATION_INT_30;
+        forward_edge.data.forward = reverse_edge.data.backward = true;
+        forward_edge.data.backward = reverse_edge.data.forward = false;
+
+        // remove parallel edges
+        while (i < edges.size() && edges[i].source == source && edges[i].target == target)
+        {
+            if (edges[i].data.forward)
+            {
+                forward_edge.data.weight = std::min(edges[i].data.weight, forward_edge.data.weight);
+                forward_edge.data.duration =
+                    std::min(edges[i].data.duration, forward_edge.data.duration);
+            }
+            if (edges[i].data.backward)
+            {
+                reverse_edge.data.weight = std::min(edges[i].data.weight, reverse_edge.data.weight);
+                reverse_edge.data.duration =
+                    std::min(edges[i].data.duration, reverse_edge.data.duration);
+            }
+            ++i;
+        }
+        // merge edges (s,t) and (t,s) into bidirectional edge
+        if (forward_edge.data.weight == reverse_edge.data.weight)
+        {
+            if ((int)forward_edge.data.weight != INVALID_EDGE_WEIGHT)
+            {
+                forward_edge.data.backward = true;
+                graph_edges.push_back(forward_edge);
+            }
+        }
+        else
+        { // insert seperate edges
+            if (((int)forward_edge.data.weight) != INVALID_EDGE_WEIGHT)
+            {
+                graph_edges.push_back(forward_edge);
+            }
+            if ((int)reverse_edge.data.weight != INVALID_EDGE_WEIGHT)
+            {
+                graph_edges.push_back(reverse_edge);
+            }
+        }
+    }
+
+    return graph_edges;
+}
+
 struct EdgeBasedGraphReader
 {
     EdgeBasedGraphReader(storage::io::FileReader &reader)
@@ -74,107 +175,6 @@ struct EdgeBasedGraphReader
     }
 
   private:
-    // Bidirectional (s,t) to (s,t) and (t,s)
-    std::vector<extractor::EdgeBasedEdge>
-    SplitBidirectionalEdges(const std::vector<extractor::EdgeBasedEdge> &edges)
-    {
-        std::vector<extractor::EdgeBasedEdge> directed;
-        directed.reserve(edges.size() * 2);
-
-        for (const auto &edge : edges)
-        {
-            directed.emplace_back(edge.source,
-                                  edge.target,
-                                  edge.edge_id,
-                                  std::max(edge.weight, 1),
-                                  edge.duration,
-                                  edge.forward,
-                                  edge.backward);
-
-            directed.emplace_back(edge.target,
-                                  edge.source,
-                                  edge.edge_id,
-                                  std::max(edge.weight, 1),
-                                  edge.duration,
-                                  edge.backward,
-                                  edge.forward);
-        }
-
-        return directed;
-    }
-
-    std::vector<EdgeBasedGraphEdge>
-    PrepareEdgesForUsageInGraph(std::vector<extractor::EdgeBasedEdge> edges)
-    {
-        std::sort(begin(edges), end(edges));
-
-        std::vector<EdgeBasedGraphEdge> graph_edges;
-        graph_edges.reserve(edges.size());
-
-        for (NodeID i = 0; i < edges.size();)
-        {
-            const NodeID source = edges[i].source;
-            const NodeID target = edges[i].target;
-
-            // remove eigenloops
-            if (source == target)
-            {
-                ++i;
-                continue;
-            }
-
-            EdgeBasedGraphEdge forward_edge;
-            EdgeBasedGraphEdge reverse_edge;
-            forward_edge.source = reverse_edge.source = source;
-            forward_edge.target = reverse_edge.target = target;
-            forward_edge.data.edge_id = reverse_edge.data.edge_id = edges[i].edge_id;
-            forward_edge.data.weight = reverse_edge.data.weight = INVALID_EDGE_WEIGHT;
-            forward_edge.data.duration = reverse_edge.data.duration = MAXIMAL_EDGE_DURATION_INT_30;
-            forward_edge.data.forward = reverse_edge.data.backward = true;
-            forward_edge.data.backward = reverse_edge.data.forward = false;
-
-            // remove parallel edges
-            while (i < edges.size() && edges[i].source == source && edges[i].target == target)
-            {
-                if (edges[i].forward)
-                {
-                    forward_edge.data.weight = std::min(edges[i].weight, forward_edge.data.weight);
-                    forward_edge.data.duration =
-                        std::min(edges[i].duration, forward_edge.data.duration);
-                }
-                if (edges[i].backward)
-                {
-                    reverse_edge.data.weight = std::min(edges[i].weight, reverse_edge.data.weight);
-                    reverse_edge.data.duration =
-                        std::min(edges[i].duration, reverse_edge.data.duration);
-                }
-                ++i;
-            }
-            // merge edges (s,t) and (t,s) into bidirectional edge
-            if (forward_edge.data.weight == reverse_edge.data.weight)
-            {
-                if ((int)forward_edge.data.weight != INVALID_EDGE_WEIGHT)
-                {
-                    forward_edge.data.backward = true;
-                    graph_edges.push_back(forward_edge);
-                }
-            }
-            else
-            { // insert seperate edges
-                if (((int)forward_edge.data.weight) != INVALID_EDGE_WEIGHT)
-                {
-                    graph_edges.push_back(forward_edge);
-                }
-                if ((int)reverse_edge.data.weight != INVALID_EDGE_WEIGHT)
-                {
-                    graph_edges.push_back(reverse_edge);
-                }
-            }
-        }
-
-        return graph_edges;
-    }
-
     std::vector<extractor::EdgeBasedEdge> edges;
     std::size_t num_nodes;
 };

include/storage/shared_datatype.hpp

@@ -64,7 +64,9 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
                                             "MLD_CELL_SOURCE_BOUNDARY",
                                             "MLD_CELL_DESTINATION_BOUNDARY",
                                             "MLD_CELLS",
-                                            "MLD_CELL_LEVEL_OFFSETS"};
+                                            "MLD_CELL_LEVEL_OFFSETS",
+                                            "MLD_GRAPH_NODE_LIST",
+                                            "MLD_GRAPH_EDGE_LIST"};
 
 struct DataLayout
 {
@@ -117,6 +119,8 @@ struct DataLayout
         MLD_CELL_DESTINATION_BOUNDARY,
         MLD_CELLS,
         MLD_CELL_LEVEL_OFFSETS,
+        MLD_GRAPH_NODE_LIST,
+        MLD_GRAPH_EDGE_LIST,
         NUM_BLOCKS
     };
 

include/storage/storage_config.hpp

@@ -71,6 +71,7 @@ struct StorageConfig final
     boost::filesystem::path turn_lane_description_path;
     boost::filesystem::path mld_partition_path;
     boost::filesystem::path mld_storage_path;
+    boost::filesystem::path edge_based_graph_path;
 };
 }
 }

include/util/shared_memory_vector_wrapper.hpp

@@ -106,6 +106,18 @@ template <typename DataT> class SharedMemoryWrapper
         return m_ptr[index];
     }
 
+    const DataT &front() const {
+        BOOST_ASSERT_MSG(m_size > 0, "invalid size");
+        return m_ptr[0];
+    }
+
+    const DataT &back() const {
+        BOOST_ASSERT_MSG(m_size > 0, "invalid size");
+        return m_ptr[m_size - 1];
+    }
+
+    auto data() const { return m_ptr; }
+
     template <typename T>
     friend void swap(SharedMemoryWrapper<T> &, SharedMemoryWrapper<T> &) noexcept;
 };

include/util/static_graph.hpp

@@ -134,8 +134,11 @@ template <typename EdgeDataT, bool UseSharedMemory = false> class StaticGraph
     StaticGraph(typename ShM<NodeArrayEntry, UseSharedMemory>::vector &nodes,
                 typename ShM<EdgeArrayEntry, UseSharedMemory>::vector &edges)
     {
+        BOOST_ASSERT(!nodes.empty());
+
         number_of_nodes = static_cast<decltype(number_of_nodes)>(nodes.size() - 1);
-        number_of_edges = static_cast<decltype(number_of_edges)>(edges.size());
+        number_of_edges = static_cast<decltype(number_of_edges)>(nodes.back().first_edge);
+        BOOST_ASSERT(number_of_edges <= edges.size());
 
         using std::swap;
         swap(node_array, nodes);

src/contractor/contractor.cpp

@@ -147,8 +147,8 @@ void CheckWeightsConsistency(
 
     for (auto &edge : edge_based_edge_list)
     {
-        BOOST_ASSERT(edge.edge_id < current_edge_data.size());
+        BOOST_ASSERT(edge.data.edge_id < current_edge_data.size());
-        auto geometry_id = current_edge_data[edge.edge_id].via_geometry;
+        auto geometry_id = current_edge_data[edge.data.edge_id].via_geometry;
         BOOST_ASSERT(geometry_id.id < geometry_indices.size());
 
         const auto &weights = geometry_id.forward ? forward_weight_list : reverse_weight_list;
@@ -157,7 +157,7 @@ void CheckWeightsConsistency(
         const auto last = weights.begin() + geometry_indices.at(geometry_id.id + 1) - 1 + shift;
         EdgeWeight weight = std::accumulate(first, last, 0);
 
-        BOOST_ASSERT(weight <= edge.weight);
+        BOOST_ASSERT(weight <= edge.data.weight);
     }
 }
 
@@ -908,8 +908,8 @@ Contractor::LoadEdgeExpandedGraph(const ContractorConfig &config,
             }
 
             // Update edge weight
-            inbuffer.weight = new_weight + turn_weight_penalty;
+            inbuffer.data.weight = new_weight + turn_weight_penalty;
-            inbuffer.duration = new_duration + turn_duration_penalty;
+            inbuffer.data.duration = new_duration + turn_duration_penalty;
         }
 
         edge_based_edge_list.emplace_back(std::move(inbuffer));

src/extractor/extractor.cpp

@@ -333,16 +333,16 @@ void Extractor::FindComponents(unsigned max_edge_id,
 
     for (const auto &edge : input_edge_list)
     {
-        BOOST_ASSERT_MSG(static_cast<unsigned int>(std::max(edge.weight, 1)) > 0,
+        BOOST_ASSERT_MSG(static_cast<unsigned int>(std::max(edge.data.weight, 1)) > 0,
                          "edge distance < 1");
         BOOST_ASSERT(edge.source <= max_edge_id);
         BOOST_ASSERT(edge.target <= max_edge_id);
-        if (edge.forward)
+        if (edge.data.forward)
         {
             edges.push_back({edge.source, edge.target});
         }
 
-        if (edge.backward)
+        if (edge.data.backward)
         {
             edges.push_back({edge.target, edge.source});
         }

src/partition/partitioner.cpp

@@ -169,6 +169,13 @@ int Partitioner::Run(const PartitionConfig &config)
                               config.minimum_cell_size * 32 * 16,
                               config.minimum_cell_size * 32 * 16 * 32});
 
+    util::Log() << "Edge-based-graph annotation:";
+    for (std::size_t level = 0; level < level_to_num_cells.size(); ++level)
+    {
+        util::Log() << "  level " << level + 1 << " #cells " << level_to_num_cells[level]
+                    << " bit size " << std::ceil(std::log2(level_to_num_cells[level] + 1));
+    }
+
     TIMER_START(packed_mlp);
     MultiLevelPartition mlp{partitions, level_to_num_cells};
     TIMER_STOP(packed_mlp);

src/storage/storage.cpp

@@ -1,12 +1,14 @@
 #include "storage/storage.hpp"
 #include "contractor/query_edge.hpp"
 #include "extractor/compressed_edge_container.hpp"
+#include "extractor/edge_based_edge.hpp"
 #include "extractor/guidance/turn_instruction.hpp"
 #include "extractor/original_edge_data.hpp"
 #include "extractor/profile_properties.hpp"
 #include "extractor/query_node.hpp"
 #include "extractor/travel_mode.hpp"
 #include "partition/cell_storage.hpp"
+#include "partition/edge_based_graph_reader.hpp"
 #include "partition/multi_level_partition.hpp"
 #include "storage/io.hpp"
 #include "storage/serialization.hpp"
@@ -54,6 +56,7 @@ using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf;
 using RTreeNode =
     util::StaticRTree<RTreeLeaf, util::ShM<util::Coordinate, true>::vector, true>::TreeNode;
 using QueryGraph = util::StaticGraph<contractor::QueryEdge::EdgeData>;
+using EdgeBasedGraph = util::StaticGraph<extractor::EdgeBasedEdge::EdgeData>;
 
 using Monitor = SharedMonitor<SharedDataTimestamp>;
 
@@ -237,6 +240,7 @@ void Storage::PopulateLayout(DataLayout &layout)
         layout.SetBlockSize<EntryClassID>(DataLayout::ENTRY_CLASSID, number_of_original_edges);
     }
 
+    if (boost::filesystem::exists(config.hsgr_data_path))
     {
         io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::VerifyFingerprint);
 
@@ -247,6 +251,12 @@ void Storage::PopulateLayout(DataLayout &layout)
         layout.SetBlockSize<QueryGraph::EdgeArrayEntry>(DataLayout::CH_GRAPH_EDGE_LIST,
                                                         hsgr_header.number_of_edges);
     }
+    else
+    {
+        layout.SetBlockSize<unsigned>(DataLayout::HSGR_CHECKSUM, 0);
+        layout.SetBlockSize<QueryGraph::NodeArrayEntry>(DataLayout::CH_GRAPH_NODE_LIST, 0);
+        layout.SetBlockSize<QueryGraph::EdgeArrayEntry>(DataLayout::CH_GRAPH_EDGE_LIST, 0);
+    }
 
     // load rsearch tree size
     {
@@ -270,11 +280,16 @@ void Storage::PopulateLayout(DataLayout &layout)
     }
 
     // load core marker size
+    if (boost::filesystem::exists(config.core_data_path))
     {
         io::FileReader core_marker_file(config.core_data_path, io::FileReader::HasNoFingerprint);
         const auto number_of_core_markers = core_marker_file.ReadElementCount32();
         layout.SetBlockSize<unsigned>(DataLayout::CH_CORE_MARKER, number_of_core_markers);
     }
+    else
+    {
+        layout.SetBlockSize<unsigned>(DataLayout::CH_CORE_MARKER, 0);
+    }
 
     // load turn weight penalties
     {
@@ -326,24 +341,25 @@ void Storage::PopulateLayout(DataLayout &layout)
                                         number_of_compressed_geometries);
     }
 
-    // load datasource sizes.  This file is optional, and it's non-fatal if it doesn't
+    // load datasource sizes. This file is optional, and it's non-fatal if it doesn't exist.
-    // exist.
+    if (boost::filesystem::exists(config.datasource_indexes_path))
     {
-        io::FileReader geometry_datasource_file(config.datasource_indexes_path,
+        io::FileReader reader(config.datasource_indexes_path, io::FileReader::HasNoFingerprint);
-                                                io::FileReader::HasNoFingerprint);
+        const auto number_of_datasources = reader.ReadElementCount64();
-        const auto number_of_compressed_datasources = geometry_datasource_file.ReadElementCount64();
+        layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST, number_of_datasources);
-        layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST,
+    }
-                                     number_of_compressed_datasources);
+    else
+    {
+        layout.SetBlockSize<uint8_t>(DataLayout::DATASOURCES_LIST, 0);
     }
 
-    // Load datasource name sizes.  This file is optional, and it's non-fatal if it doesn't
+    // Load datasource name sizes.  This file is optional, and it's non-fatal if it doesn't exist
-    // exist
+    if (boost::filesystem::exists(config.datasource_names_path))
     {
-        io::FileReader datasource_names_file(config.datasource_names_path,
+        io::FileReader reader(config.datasource_names_path, io::FileReader::HasNoFingerprint);
-                                             io::FileReader::HasNoFingerprint);
 
         const serialization::DatasourceNamesData datasource_names_data =
-            serialization::readDatasourceNames(datasource_names_file);
+            serialization::readDatasourceNames(reader);
 
         layout.SetBlockSize<char>(DataLayout::DATASOURCE_NAME_DATA,
                                   datasource_names_data.names.size());
@@ -352,6 +368,12 @@ void Storage::PopulateLayout(DataLayout &layout)
         layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_LENGTHS,
                                          datasource_names_data.lengths.size());
     }
+    else
+    {
+        layout.SetBlockSize<char>(DataLayout::DATASOURCE_NAME_DATA, 0);
+        layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_OFFSETS, 0);
+        layout.SetBlockSize<std::size_t>(DataLayout::DATASOURCE_NAME_LENGTHS, 0);
+    }
 
     {
         io::FileReader intersection_file(config.intersection_class_path,
@@ -446,6 +468,25 @@ void Storage::PopulateLayout(DataLayout &layout)
             layout.SetBlockSize<char>(DataLayout::MLD_CELLS, 0);
             layout.SetBlockSize<char>(DataLayout::MLD_CELL_LEVEL_OFFSETS, 0);
         }
+
+        if (boost::filesystem::exists(config.edge_based_graph_path))
+        {
+            io::FileReader ebg_file(config.edge_based_graph_path,
+                                    io::FileReader::VerifyFingerprint);
+
+            const auto num_edges = ebg_file.ReadElementCount64();
+            const auto num_nodes = ebg_file.ReadOne<EdgeID>() + 1;
+
+            layout.SetBlockSize<EdgeBasedGraph::NodeArrayEntry>(DataLayout::MLD_GRAPH_NODE_LIST,
+                                                                num_nodes + 1);
+            layout.SetBlockSize<EdgeBasedGraph::EdgeArrayEntry>(DataLayout::MLD_GRAPH_EDGE_LIST,
+                                                                2 * num_edges);
+        }
+        else
+        {
+            layout.SetBlockSize<char>(DataLayout::MLD_GRAPH_NODE_LIST, 0);
+            layout.SetBlockSize<char>(DataLayout::MLD_GRAPH_EDGE_LIST, 0);
+        }
     }
 }
 
@@ -456,6 +497,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // read actual data into shared memory object //
 
     // Load the HSGR file
+    if (boost::filesystem::exists(config.hsgr_data_path))
     {
         io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::VerifyFingerprint);
         auto hsgr_header = serialization::readHSGRHeader(hsgr_file);
@@ -479,6 +521,14 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
                                 graph_edge_list_ptr,
                                 hsgr_header.number_of_edges);
     }
+    else
+    {
+        layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
+        layout.GetBlockPtr<QueryGraph::NodeArrayEntry, true>(memory_ptr,
+                                                             DataLayout::CH_GRAPH_NODE_LIST);
+        layout.GetBlockPtr<QueryGraph::EdgeArrayEntry, true>(memory_ptr,
+                                                             DataLayout::CH_GRAPH_EDGE_LIST);
+    }
 
     // store the filename of the on-disk portion of the RTree
     {
@@ -643,6 +693,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
         geometry_input_file.ReadInto(geometries_rev_duration_list_ptr, geometry_node_lists_count);
     }
 
+    if (boost::filesystem::exists(config.datasource_indexes_path))
     {
         io::FileReader geometry_datasource_file(config.datasource_indexes_path,
                                                 io::FileReader::HasNoFingerprint);
@@ -657,9 +708,13 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
                 geometry_datasource_file, datasources_list_ptr, number_of_compressed_datasources);
         }
     }
-
+    else
+    {
+        layout.GetBlockPtr<uint8_t, true>(memory_ptr, DataLayout::DATASOURCES_LIST);
+    }
+
+    if (boost::filesystem::exists(config.datasource_names_path))
     {
-        /* Load names */
         io::FileReader datasource_names_file(config.datasource_names_path,
                                              io::FileReader::HasNoFingerprint);
 
@@ -706,6 +761,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
                       datasource_name_lengths_ptr);
         }
     }
+    else
+    {
+        layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::DATASOURCE_NAME_DATA);
+        layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_OFFSETS);
+        layout.GetBlockPtr<std::size_t, true>(memory_ptr, DataLayout::DATASOURCE_NAME_LENGTHS);
+    }
 
     // Loading list of coordinates
     {
@@ -768,6 +829,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
         tree_node_file.ReadInto(rtree_ptr, layout.num_entries[DataLayout::R_SEARCH_TREE]);
     }
 
+    if (boost::filesystem::exists(config.core_data_path))
     {
         io::FileReader core_marker_file(config.core_data_path, io::FileReader::HasNoFingerprint);
         const auto number_of_core_markers = core_marker_file.ReadElementCount32();
@@ -939,6 +1001,40 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
             reader.ReadInto(mld_cell_level_offsets_ptr,
                             layout.GetBlockEntries(DataLayout::MLD_CELL_LEVEL_OFFSETS));
         }
+
+        if (boost::filesystem::exists(config.edge_based_graph_path))
+        {
+            io::FileReader reader(config.edge_based_graph_path, io::FileReader::VerifyFingerprint);
+
+            const auto number_of_edges = reader.ReadElementCount64();
+            const auto number_of_nodes = reader.ReadOne<EdgeID>() + 1;
+            std::vector<extractor::EdgeBasedEdge> original_edges(number_of_edges);
+            reader.ReadInto(original_edges);
+
+            original_edges = partition::SplitBidirectionalEdges(std::move(original_edges));
+            auto edges = partition::PrepareEdgesForUsageInGraph(std::move(original_edges));
+            BOOST_ASSERT(edges.size() <= 2 * number_of_edges);
+
+            auto nodes_ptr = layout.GetBlockPtr<EdgeBasedGraph::NodeArrayEntry, true>(
+                memory_ptr, DataLayout::MLD_GRAPH_NODE_LIST);
+            auto edges_ptr = layout.GetBlockPtr<EdgeBasedGraph::EdgeArrayEntry, true>(
+                memory_ptr, DataLayout::MLD_GRAPH_EDGE_LIST);
+
+            EdgeBasedGraph::EdgeIterator edge = 0;
+            for (const auto node : util::irange(0u, number_of_nodes + 1))
+            {
+                EdgeBasedGraph::EdgeIterator last_edge = edge;
+                while (edge < edges.size() && edges[edge].source == node)
+                {
+                    edges_ptr[edge].target = edges[edge].target;
+                    edges_ptr[edge].data = edges[edge].data;
+                    ++edge;
+                }
+                nodes_ptr[node].first_edge = last_edge;
+            }
+
+            BOOST_ASSERT(edge == edges.size());
+        }
     }
 }
 }

src/storage/storage_config.cpp

@@ -7,6 +7,22 @@ namespace osrm
 {
 namespace storage
 {
+namespace
+{
+bool CheckFileList(const std::vector<boost::filesystem::path> &files)
+{
+    bool success = true;
+    for (auto &path : files)
+    {
+        if (!boost::filesystem::is_regular_file(path))
+        {
+            util::Log(logWARNING) << "Missing/Broken File: " << path.string();
+            success = false;
+        }
+    }
+    return success;
+}
+}
 
 StorageConfig::StorageConfig(const boost::filesystem::path &base)
     : ram_index_path{base.string() + ".ramIndex"}, file_index_path{base.string() + ".fileIndex"},
@@ -20,40 +36,40 @@ StorageConfig::StorageConfig(const boost::filesystem::path &base)
       names_data_path{base.string() + ".names"}, properties_path{base.string() + ".properties"},
       intersection_class_path{base.string() + ".icd"}, turn_lane_data_path{base.string() + ".tld"},
       turn_lane_description_path{base.string() + ".tls"},
-      mld_partition_path{base.string() + ".partition"}, mld_storage_path{base.string() + ".cells"}
+      mld_partition_path{base.string() + ".partition"}, mld_storage_path{base.string() + ".cells"},
+      edge_based_graph_path{base.string() + ".ebg"}
 {
 }
 
 bool StorageConfig::IsValid() const
 {
     const constexpr auto num_files = 15;
-    const boost::filesystem::path paths[num_files] = {ram_index_path,
-                                                      file_index_path,
-                                                      hsgr_data_path,
-                                                      nodes_data_path,
-                                                      edges_data_path,
-                                                      core_data_path,
-                                                      geometries_path,
-                                                      timestamp_path,
-                                                      turn_weight_penalties_path,
-                                                      turn_duration_penalties_path,
-                                                      datasource_names_path,
-                                                      datasource_indexes_path,
-                                                      names_data_path,
-                                                      properties_path,
-                                                      intersection_class_path};
 
-    bool success = true;
+    // Common files
-    for (auto path = paths; path != paths + num_files; ++path)
+    if (!CheckFileList({ram_index_path,
+                        file_index_path,
+                        nodes_data_path,
+                        edges_data_path,
+                        geometries_path,
+                        timestamp_path,
+                        turn_weight_penalties_path,
+                        turn_duration_penalties_path,
+                        names_data_path,
+                        properties_path,
+                        intersection_class_path}))
     {
-        if (!boost::filesystem::is_regular_file(*path))
+        return false;
-        {
-            util::Log(logWARNING) << "Missing/Broken File: " << path->string();
-            success = false;
-        }
     }
 
-    return success;
+    // TODO: add algorithm checks
+
+    // CH files
+    CheckFileList({hsgr_data_path, core_data_path, datasource_names_path, datasource_indexes_path});
+
+    // MLD files
+    CheckFileList({mld_partition_path, mld_storage_path, edge_based_graph_path});
+
+    return true;
 }
 }
 }

unit_tests/util/static_graph.cpp

@@ -42,7 +42,7 @@ template <unsigned NUM_NODES, unsigned NUM_EDGES> struct RandomArrayEntryFixture
         }
         std::sort(offsets.begin(), offsets.end());
         // add sentinel
-        offsets.push_back(offsets.back());
+        offsets.push_back(TEST_NUM_EDGES);
 
         // extract interval lengths
         for (unsigned i = 0; i < offsets.size() - 1; i++)