From f42136637d6e607495da49c17898345337279985 Mon Sep 17 00:00:00 2001 From: Michael Krasnyk Date: Mon, 6 Mar 2017 23:00:11 +0100 Subject: [PATCH] Add edge-based graph loading in MLD facade --- .../contractor/graph_contractor_adaptors.hpp | 24 +-- .../datafacade/algorithm_datafacade.hpp | 20 ++ .../contiguous_internalmem_datafacade.hpp | 55 +++++ include/extractor/edge_based_edge.hpp | 44 ++-- include/partition/edge_based_graph_reader.hpp | 204 +++++++++--------- include/storage/shared_datatype.hpp | 6 +- include/storage/storage_config.hpp | 1 + include/util/shared_memory_vector_wrapper.hpp | 12 ++ include/util/static_graph.hpp | 5 +- src/contractor/contractor.cpp | 10 +- src/extractor/extractor.cpp | 6 +- src/partition/partitioner.cpp | 7 + src/storage/storage.cpp | 124 +++++++++-- src/storage/storage_config.cpp | 64 +++--- unit_tests/util/static_graph.cpp | 2 +- 15 files changed, 406 insertions(+), 178 deletions(-) diff --git a/include/contractor/graph_contractor_adaptors.hpp b/include/contractor/graph_contractor_adaptors.hpp index 0ebad30d5..969e78c85 100644 --- a/include/contractor/graph_contractor_adaptors.hpp +++ b/include/contractor/graph_contractor_adaptors.hpp @@ -22,33 +22,33 @@ std::vector adaptToContractorInput(InputEdgeContainer input_edge { #ifndef NDEBUG const unsigned int constexpr DAY_IN_DECI_SECONDS = 24 * 60 * 60 * 10; - if (static_cast(std::max(input_edge.weight, 1)) > DAY_IN_DECI_SECONDS) + if (static_cast(std::max(input_edge.data.weight, 1)) > DAY_IN_DECI_SECONDS) { util::Log(logWARNING) << "Edge weight large -> " - << static_cast(std::max(input_edge.weight, 1)) + << static_cast(std::max(input_edge.data.weight, 1)) << " : " << static_cast(input_edge.source) << " -> " << static_cast(input_edge.target); } #endif edges.emplace_back(input_edge.source, input_edge.target, - std::max(input_edge.weight, 1), - input_edge.duration, + std::max(input_edge.data.weight, 1), + input_edge.data.duration, 1, - input_edge.edge_id, + input_edge.data.edge_id, false, - input_edge.forward ? true : false, - input_edge.backward ? true : false); + input_edge.data.forward ? true : false, + input_edge.data.backward ? true : false); edges.emplace_back(input_edge.target, input_edge.source, - std::max(input_edge.weight, 1), - input_edge.duration, + std::max(input_edge.data.weight, 1), + input_edge.data.duration, 1, - input_edge.edge_id, + input_edge.data.edge_id, false, - input_edge.backward ? true : false, - input_edge.forward ? true : false); + input_edge.data.backward ? true : false, + input_edge.data.forward ? true : false); } // FIXME not sure if we need this edges.shrink_to_fit(); diff --git a/include/engine/datafacade/algorithm_datafacade.hpp b/include/engine/datafacade/algorithm_datafacade.hpp index 70cf74ffb..5608e9251 100644 --- a/include/engine/datafacade/algorithm_datafacade.hpp +++ b/include/engine/datafacade/algorithm_datafacade.hpp @@ -2,6 +2,7 @@ #define OSRM_ENGINE_DATAFACADE_ALGORITHM_DATAFACADE_HPP #include "contractor/query_edge.hpp" +#include "extractor/edge_based_edge.hpp" #include "engine/algorithm.hpp" #include "partition/cell_storage.hpp" @@ -66,6 +67,25 @@ template <> class AlgorithmDataFacade template <> class AlgorithmDataFacade { public: + using EdgeData = extractor::EdgeBasedEdge::EdgeData; + + // search graph access + virtual unsigned GetNumberOfNodes() const = 0; + + virtual unsigned GetNumberOfEdges() const = 0; + + virtual unsigned GetOutDegree(const NodeID n) const = 0; + + virtual NodeID GetTarget(const EdgeID e) const = 0; + + virtual const EdgeData &GetEdgeData(const EdgeID e) const = 0; + + virtual EdgeID BeginEdges(const NodeID n) const = 0; + + virtual EdgeID EndEdges(const NodeID n) const = 0; + + virtual EdgeRange GetAdjacentEdgeRange(const NodeID node) const = 0; + virtual const partition::MultiLevelPartitionView &GetMultiLevelPartition() const = 0; virtual const partition::CellStorageView &GetCellStorage() const = 0; diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp index fe1f52339..6f0049b3d 100644 --- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp +++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp @@ -1200,12 +1200,67 @@ class ContiguousInternalMemoryDataFacade : public ContiguousInternalMemoryDataFacadeBase, public ContiguousInternalMemoryAlgorithmDataFacade { + private: + using QueryGraph = util::StaticGraph; + using GraphNode = QueryGraph::NodeArrayEntry; + using GraphEdge = QueryGraph::EdgeArrayEntry; + + std::unique_ptr m_query_graph; + + void InitializeGraphPointer(storage::DataLayout &data_layout, char *memory_block) + { + auto graph_nodes_ptr = data_layout.GetBlockPtr( + memory_block, storage::DataLayout::MLD_GRAPH_NODE_LIST); + + auto graph_edges_ptr = data_layout.GetBlockPtr( + memory_block, storage::DataLayout::MLD_GRAPH_EDGE_LIST); + + util::ShM::vector node_list( + graph_nodes_ptr, data_layout.num_entries[storage::DataLayout::MLD_GRAPH_NODE_LIST]); + util::ShM::vector edge_list( + graph_edges_ptr, data_layout.num_entries[storage::DataLayout::MLD_GRAPH_EDGE_LIST]); + + m_query_graph.reset(new QueryGraph(node_list, edge_list)); + } + public: ContiguousInternalMemoryDataFacade(std::shared_ptr allocator) : ContiguousInternalMemoryDataFacadeBase(allocator), ContiguousInternalMemoryAlgorithmDataFacade(allocator) { + InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory()); + } + + void InitializeInternalPointers(storage::DataLayout &data_layout, char *memory_block) + { + InitializeGraphPointer(data_layout, memory_block); + } + + // search graph access + unsigned GetNumberOfNodes() const override final { return m_query_graph->GetNumberOfNodes(); } + + unsigned GetNumberOfEdges() const override final { return m_query_graph->GetNumberOfEdges(); } + + unsigned GetOutDegree(const NodeID n) const override final + { + return m_query_graph->GetOutDegree(n); + } + + NodeID GetTarget(const EdgeID e) const override final { return m_query_graph->GetTarget(e); } + + EdgeData &GetEdgeData(const EdgeID e) const override final + { + return m_query_graph->GetEdgeData(e); + } + + EdgeID BeginEdges(const NodeID n) const override final { return m_query_graph->BeginEdges(n); } + + EdgeID EndEdges(const NodeID n) const override final { return m_query_graph->EndEdges(n); } + + EdgeRange GetAdjacentEdgeRange(const NodeID node) const override final + { + return m_query_graph->GetAdjacentEdgeRange(node); } }; } diff --git a/include/extractor/edge_based_edge.hpp b/include/extractor/edge_based_edge.hpp index 06bd5f315..cbc10308d 100644 --- a/include/extractor/edge_based_edge.hpp +++ b/include/extractor/edge_based_edge.hpp @@ -29,11 +29,29 @@ struct EdgeBasedEdge NodeID source; NodeID target; - NodeID edge_id; - EdgeWeight weight; - EdgeWeight duration : 30; - std::uint32_t forward : 1; - std::uint32_t backward : 1; + + struct EdgeData + { + EdgeData() : edge_id(0), weight(0), duration(0), forward(false), backward(false) {} + + EdgeData(const NodeID edge_id, + const EdgeWeight weight, + const EdgeWeight duration, + const bool forward, + const bool backward) + : edge_id(edge_id), weight(weight), duration(duration), forward(forward), + backward(backward) + { + } + + NodeID edge_id; + EdgeWeight weight; + EdgeWeight duration : 30; + std::uint32_t forward : 1; + std::uint32_t backward : 1; + + auto is_unidirectional() const { return !forward || !backward; } + } data; }; static_assert(sizeof(extractor::EdgeBasedEdge) == 20, "Size of extractor::EdgeBasedEdge type is " @@ -42,10 +60,7 @@ static_assert(sizeof(extractor::EdgeBasedEdge) == 20, // Impl. -inline EdgeBasedEdge::EdgeBasedEdge() - : source(0), target(0), edge_id(0), weight(0), duration(0), forward(false), backward(false) -{ -} +inline EdgeBasedEdge::EdgeBasedEdge() : source(0), target(0) {} inline EdgeBasedEdge::EdgeBasedEdge(const NodeID source, const NodeID target, @@ -54,19 +69,18 @@ inline EdgeBasedEdge::EdgeBasedEdge(const NodeID source, const EdgeWeight duration, const bool forward, const bool backward) - : source(source), target(target), edge_id(edge_id), weight(weight), duration(duration), - forward(forward), backward(backward) + : source(source), target(target), data{edge_id, weight, duration, forward, backward} { } inline bool EdgeBasedEdge::operator<(const EdgeBasedEdge &other) const { - const auto unidirectional = (!forward || !backward); - const auto other_is_unidirectional = (!other.forward || !other.backward); + const auto unidirectional = data.is_unidirectional(); + const auto other_is_unidirectional = other.data.is_unidirectional(); // if all items are the same, we want to keep bidirectional edges. due to the `<` operator, // preferring 0 (false) over 1 (true), we need to compare the inverse of `bidirectional` - return std::tie(source, target, weight, unidirectional) < - std::tie(other.source, other.target, other.weight, other_is_unidirectional); + return std::tie(source, target, data.weight, unidirectional) < + std::tie(other.source, other.target, other.data.weight, other_is_unidirectional); } } // ns extractor } // ns osrm diff --git a/include/partition/edge_based_graph_reader.hpp b/include/partition/edge_based_graph_reader.hpp index 94707fa04..390ab611b 100644 --- a/include/partition/edge_based_graph_reader.hpp +++ b/include/partition/edge_based_graph_reader.hpp @@ -19,7 +19,7 @@ namespace osrm namespace partition { -struct EdgeBasedGraphEdgeData : extractor::EdgeBasedEdge +struct EdgeBasedGraphEdgeData : extractor::EdgeBasedEdge::EdgeData { // We need to write out the full edge based graph again. @@ -38,6 +38,107 @@ struct EdgeBasedGraphEdge : EdgeBasedGraph::InputEdge using Base::Base; }; +// Bidirectional (s,t) to (s,t) and (t,s) +std::vector +SplitBidirectionalEdges(const std::vector &edges) +{ + std::vector directed; + directed.reserve(edges.size() * 2); + + for (const auto &edge : edges) + { + directed.emplace_back(edge.source, + edge.target, + edge.data.edge_id, + std::max(edge.data.weight, 1), + edge.data.duration, + edge.data.forward, + edge.data.backward); + + directed.emplace_back(edge.target, + edge.source, + edge.data.edge_id, + std::max(edge.data.weight, 1), + edge.data.duration, + edge.data.backward, + edge.data.forward); + } + + return directed; +} + +std::vector +PrepareEdgesForUsageInGraph(std::vector edges) +{ + std::sort(begin(edges), end(edges)); + + std::vector graph_edges; + graph_edges.reserve(edges.size()); + + for (NodeID i = 0; i < edges.size();) + { + const NodeID source = edges[i].source; + const NodeID target = edges[i].target; + + // remove eigenloops + if (source == target) + { + ++i; + continue; + } + + EdgeBasedGraphEdge forward_edge; + EdgeBasedGraphEdge reverse_edge; + forward_edge.source = reverse_edge.source = source; + forward_edge.target = reverse_edge.target = target; + forward_edge.data.edge_id = reverse_edge.data.edge_id = edges[i].data.edge_id; + forward_edge.data.weight = reverse_edge.data.weight = INVALID_EDGE_WEIGHT; + forward_edge.data.duration = reverse_edge.data.duration = MAXIMAL_EDGE_DURATION_INT_30; + forward_edge.data.forward = reverse_edge.data.backward = true; + forward_edge.data.backward = reverse_edge.data.forward = false; + + // remove parallel edges + while (i < edges.size() && edges[i].source == source && edges[i].target == target) + { + if (edges[i].data.forward) + { + forward_edge.data.weight = std::min(edges[i].data.weight, forward_edge.data.weight); + forward_edge.data.duration = + std::min(edges[i].data.duration, forward_edge.data.duration); + } + if (edges[i].data.backward) + { + reverse_edge.data.weight = std::min(edges[i].data.weight, reverse_edge.data.weight); + reverse_edge.data.duration = + std::min(edges[i].data.duration, reverse_edge.data.duration); + } + ++i; + } + // merge edges (s,t) and (t,s) into bidirectional edge + if (forward_edge.data.weight == reverse_edge.data.weight) + { + if ((int)forward_edge.data.weight != INVALID_EDGE_WEIGHT) + { + forward_edge.data.backward = true; + graph_edges.push_back(forward_edge); + } + } + else + { // insert seperate edges + if (((int)forward_edge.data.weight) != INVALID_EDGE_WEIGHT) + { + graph_edges.push_back(forward_edge); + } + if ((int)reverse_edge.data.weight != INVALID_EDGE_WEIGHT) + { + graph_edges.push_back(reverse_edge); + } + } + } + + return graph_edges; +} + struct EdgeBasedGraphReader { EdgeBasedGraphReader(storage::io::FileReader &reader) @@ -74,107 +175,6 @@ struct EdgeBasedGraphReader } private: - // Bidirectional (s,t) to (s,t) and (t,s) - std::vector - SplitBidirectionalEdges(const std::vector &edges) - { - std::vector directed; - directed.reserve(edges.size() * 2); - - for (const auto &edge : edges) - { - directed.emplace_back(edge.source, - edge.target, - edge.edge_id, - std::max(edge.weight, 1), - edge.duration, - edge.forward, - edge.backward); - - directed.emplace_back(edge.target, - edge.source, - edge.edge_id, - std::max(edge.weight, 1), - edge.duration, - edge.backward, - edge.forward); - } - - return directed; - } - - std::vector - PrepareEdgesForUsageInGraph(std::vector edges) - { - std::sort(begin(edges), end(edges)); - - std::vector graph_edges; - graph_edges.reserve(edges.size()); - - for (NodeID i = 0; i < edges.size();) - { - const NodeID source = edges[i].source; - const NodeID target = edges[i].target; - - // remove eigenloops - if (source == target) - { - ++i; - continue; - } - - EdgeBasedGraphEdge forward_edge; - EdgeBasedGraphEdge reverse_edge; - forward_edge.source = reverse_edge.source = source; - forward_edge.target = reverse_edge.target = target; - forward_edge.data.edge_id = reverse_edge.data.edge_id = edges[i].edge_id; - forward_edge.data.weight = reverse_edge.data.weight = INVALID_EDGE_WEIGHT; - forward_edge.data.duration = reverse_edge.data.duration = MAXIMAL_EDGE_DURATION_INT_30; - forward_edge.data.forward = reverse_edge.data.backward = true; - forward_edge.data.backward = reverse_edge.data.forward = false; - - // remove parallel edges - while (i < edges.size() && edges[i].source == source && edges[i].target == target) - { - if (edges[i].forward) - { - forward_edge.data.weight = std::min(edges[i].weight, forward_edge.data.weight); - forward_edge.data.duration = - std::min(edges[i].duration, forward_edge.data.duration); - } - if (edges[i].backward) - { - reverse_edge.data.weight = std::min(edges[i].weight, reverse_edge.data.weight); - reverse_edge.data.duration = - std::min(edges[i].duration, reverse_edge.data.duration); - } - ++i; - } - // merge edges (s,t) and (t,s) into bidirectional edge - if (forward_edge.data.weight == reverse_edge.data.weight) - { - if ((int)forward_edge.data.weight != INVALID_EDGE_WEIGHT) - { - forward_edge.data.backward = true; - graph_edges.push_back(forward_edge); - } - } - else - { // insert seperate edges - if (((int)forward_edge.data.weight) != INVALID_EDGE_WEIGHT) - { - graph_edges.push_back(forward_edge); - } - if ((int)reverse_edge.data.weight != INVALID_EDGE_WEIGHT) - { - graph_edges.push_back(reverse_edge); - } - } - } - - return graph_edges; - } - std::vector edges; std::size_t num_nodes; }; diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp index 9798e0bb0..cb1f75c10 100644 --- a/include/storage/shared_datatype.hpp +++ b/include/storage/shared_datatype.hpp @@ -64,7 +64,9 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA", "MLD_CELL_SOURCE_BOUNDARY", "MLD_CELL_DESTINATION_BOUNDARY", "MLD_CELLS", - "MLD_CELL_LEVEL_OFFSETS"}; + "MLD_CELL_LEVEL_OFFSETS", + "MLD_GRAPH_NODE_LIST", + "MLD_GRAPH_EDGE_LIST"}; struct DataLayout { @@ -117,6 +119,8 @@ struct DataLayout MLD_CELL_DESTINATION_BOUNDARY, MLD_CELLS, MLD_CELL_LEVEL_OFFSETS, + MLD_GRAPH_NODE_LIST, + MLD_GRAPH_EDGE_LIST, NUM_BLOCKS }; diff --git a/include/storage/storage_config.hpp b/include/storage/storage_config.hpp index 397187d52..d241e249a 100644 --- a/include/storage/storage_config.hpp +++ b/include/storage/storage_config.hpp @@ -71,6 +71,7 @@ struct StorageConfig final boost::filesystem::path turn_lane_description_path; boost::filesystem::path mld_partition_path; boost::filesystem::path mld_storage_path; + boost::filesystem::path edge_based_graph_path; }; } } diff --git a/include/util/shared_memory_vector_wrapper.hpp b/include/util/shared_memory_vector_wrapper.hpp index a86ac4586..9e0fa6228 100644 --- a/include/util/shared_memory_vector_wrapper.hpp +++ b/include/util/shared_memory_vector_wrapper.hpp @@ -106,6 +106,18 @@ template class SharedMemoryWrapper return m_ptr[index]; } + const DataT &front() const { + BOOST_ASSERT_MSG(m_size > 0, "invalid size"); + return m_ptr[0]; + } + + const DataT &back() const { + BOOST_ASSERT_MSG(m_size > 0, "invalid size"); + return m_ptr[m_size - 1]; + } + + auto data() const { return m_ptr; } + template friend void swap(SharedMemoryWrapper &, SharedMemoryWrapper &) noexcept; }; diff --git a/include/util/static_graph.hpp b/include/util/static_graph.hpp index 9a0120844..7f552064e 100644 --- a/include/util/static_graph.hpp +++ b/include/util/static_graph.hpp @@ -134,8 +134,11 @@ template class StaticGraph StaticGraph(typename ShM::vector &nodes, typename ShM::vector &edges) { + BOOST_ASSERT(!nodes.empty()); + number_of_nodes = static_cast(nodes.size() - 1); - number_of_edges = static_cast(edges.size()); + number_of_edges = static_cast(nodes.back().first_edge); + BOOST_ASSERT(number_of_edges <= edges.size()); using std::swap; swap(node_array, nodes); diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp index bf9741bb7..b0f0e0988 100644 --- a/src/contractor/contractor.cpp +++ b/src/contractor/contractor.cpp @@ -147,8 +147,8 @@ void CheckWeightsConsistency( for (auto &edge : edge_based_edge_list) { - BOOST_ASSERT(edge.edge_id < current_edge_data.size()); - auto geometry_id = current_edge_data[edge.edge_id].via_geometry; + BOOST_ASSERT(edge.data.edge_id < current_edge_data.size()); + auto geometry_id = current_edge_data[edge.data.edge_id].via_geometry; BOOST_ASSERT(geometry_id.id < geometry_indices.size()); const auto &weights = geometry_id.forward ? forward_weight_list : reverse_weight_list; @@ -157,7 +157,7 @@ void CheckWeightsConsistency( const auto last = weights.begin() + geometry_indices.at(geometry_id.id + 1) - 1 + shift; EdgeWeight weight = std::accumulate(first, last, 0); - BOOST_ASSERT(weight <= edge.weight); + BOOST_ASSERT(weight <= edge.data.weight); } } @@ -908,8 +908,8 @@ Contractor::LoadEdgeExpandedGraph(const ContractorConfig &config, } // Update edge weight - inbuffer.weight = new_weight + turn_weight_penalty; - inbuffer.duration = new_duration + turn_duration_penalty; + inbuffer.data.weight = new_weight + turn_weight_penalty; + inbuffer.data.duration = new_duration + turn_duration_penalty; } edge_based_edge_list.emplace_back(std::move(inbuffer)); diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 18e9eb1dd..fc0d482e5 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -333,16 +333,16 @@ void Extractor::FindComponents(unsigned max_edge_id, for (const auto &edge : input_edge_list) { - BOOST_ASSERT_MSG(static_cast(std::max(edge.weight, 1)) > 0, + BOOST_ASSERT_MSG(static_cast(std::max(edge.data.weight, 1)) > 0, "edge distance < 1"); BOOST_ASSERT(edge.source <= max_edge_id); BOOST_ASSERT(edge.target <= max_edge_id); - if (edge.forward) + if (edge.data.forward) { edges.push_back({edge.source, edge.target}); } - if (edge.backward) + if (edge.data.backward) { edges.push_back({edge.target, edge.source}); } diff --git a/src/partition/partitioner.cpp b/src/partition/partitioner.cpp index c9c1e7721..b0f9d3b7a 100644 --- a/src/partition/partitioner.cpp +++ b/src/partition/partitioner.cpp @@ -169,6 +169,13 @@ int Partitioner::Run(const PartitionConfig &config) config.minimum_cell_size * 32 * 16, config.minimum_cell_size * 32 * 16 * 32}); + util::Log() << "Edge-based-graph annotation:"; + for (std::size_t level = 0; level < level_to_num_cells.size(); ++level) + { + util::Log() << " level " << level + 1 << " #cells " << level_to_num_cells[level] + << " bit size " << std::ceil(std::log2(level_to_num_cells[level] + 1)); + } + TIMER_START(packed_mlp); MultiLevelPartition mlp{partitions, level_to_num_cells}; TIMER_STOP(packed_mlp); diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index ac930e57f..30342f3a5 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -1,12 +1,14 @@ #include "storage/storage.hpp" #include "contractor/query_edge.hpp" #include "extractor/compressed_edge_container.hpp" +#include "extractor/edge_based_edge.hpp" #include "extractor/guidance/turn_instruction.hpp" #include "extractor/original_edge_data.hpp" #include "extractor/profile_properties.hpp" #include "extractor/query_node.hpp" #include "extractor/travel_mode.hpp" #include "partition/cell_storage.hpp" +#include "partition/edge_based_graph_reader.hpp" #include "partition/multi_level_partition.hpp" #include "storage/io.hpp" #include "storage/serialization.hpp" @@ -54,6 +56,7 @@ using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf; using RTreeNode = util::StaticRTree::vector, true>::TreeNode; using QueryGraph = util::StaticGraph; +using EdgeBasedGraph = util::StaticGraph; using Monitor = SharedMonitor; @@ -237,6 +240,7 @@ void Storage::PopulateLayout(DataLayout &layout) layout.SetBlockSize(DataLayout::ENTRY_CLASSID, number_of_original_edges); } + if (boost::filesystem::exists(config.hsgr_data_path)) { io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::VerifyFingerprint); @@ -247,6 +251,12 @@ void Storage::PopulateLayout(DataLayout &layout) layout.SetBlockSize(DataLayout::CH_GRAPH_EDGE_LIST, hsgr_header.number_of_edges); } + else + { + layout.SetBlockSize(DataLayout::HSGR_CHECKSUM, 0); + layout.SetBlockSize(DataLayout::CH_GRAPH_NODE_LIST, 0); + layout.SetBlockSize(DataLayout::CH_GRAPH_EDGE_LIST, 0); + } // load rsearch tree size { @@ -270,11 +280,16 @@ void Storage::PopulateLayout(DataLayout &layout) } // load core marker size + if (boost::filesystem::exists(config.core_data_path)) { io::FileReader core_marker_file(config.core_data_path, io::FileReader::HasNoFingerprint); const auto number_of_core_markers = core_marker_file.ReadElementCount32(); layout.SetBlockSize(DataLayout::CH_CORE_MARKER, number_of_core_markers); } + else + { + layout.SetBlockSize(DataLayout::CH_CORE_MARKER, 0); + } // load turn weight penalties { @@ -326,24 +341,25 @@ void Storage::PopulateLayout(DataLayout &layout) number_of_compressed_geometries); } - // load datasource sizes. This file is optional, and it's non-fatal if it doesn't - // exist. + // load datasource sizes. This file is optional, and it's non-fatal if it doesn't exist. + if (boost::filesystem::exists(config.datasource_indexes_path)) { - io::FileReader geometry_datasource_file(config.datasource_indexes_path, - io::FileReader::HasNoFingerprint); - const auto number_of_compressed_datasources = geometry_datasource_file.ReadElementCount64(); - layout.SetBlockSize(DataLayout::DATASOURCES_LIST, - number_of_compressed_datasources); + io::FileReader reader(config.datasource_indexes_path, io::FileReader::HasNoFingerprint); + const auto number_of_datasources = reader.ReadElementCount64(); + layout.SetBlockSize(DataLayout::DATASOURCES_LIST, number_of_datasources); + } + else + { + layout.SetBlockSize(DataLayout::DATASOURCES_LIST, 0); } - // Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't - // exist + // Load datasource name sizes. This file is optional, and it's non-fatal if it doesn't exist + if (boost::filesystem::exists(config.datasource_names_path)) { - io::FileReader datasource_names_file(config.datasource_names_path, - io::FileReader::HasNoFingerprint); + io::FileReader reader(config.datasource_names_path, io::FileReader::HasNoFingerprint); const serialization::DatasourceNamesData datasource_names_data = - serialization::readDatasourceNames(datasource_names_file); + serialization::readDatasourceNames(reader); layout.SetBlockSize(DataLayout::DATASOURCE_NAME_DATA, datasource_names_data.names.size()); @@ -352,6 +368,12 @@ void Storage::PopulateLayout(DataLayout &layout) layout.SetBlockSize(DataLayout::DATASOURCE_NAME_LENGTHS, datasource_names_data.lengths.size()); } + else + { + layout.SetBlockSize(DataLayout::DATASOURCE_NAME_DATA, 0); + layout.SetBlockSize(DataLayout::DATASOURCE_NAME_OFFSETS, 0); + layout.SetBlockSize(DataLayout::DATASOURCE_NAME_LENGTHS, 0); + } { io::FileReader intersection_file(config.intersection_class_path, @@ -446,6 +468,25 @@ void Storage::PopulateLayout(DataLayout &layout) layout.SetBlockSize(DataLayout::MLD_CELLS, 0); layout.SetBlockSize(DataLayout::MLD_CELL_LEVEL_OFFSETS, 0); } + + if (boost::filesystem::exists(config.edge_based_graph_path)) + { + io::FileReader ebg_file(config.edge_based_graph_path, + io::FileReader::VerifyFingerprint); + + const auto num_edges = ebg_file.ReadElementCount64(); + const auto num_nodes = ebg_file.ReadOne() + 1; + + layout.SetBlockSize(DataLayout::MLD_GRAPH_NODE_LIST, + num_nodes + 1); + layout.SetBlockSize(DataLayout::MLD_GRAPH_EDGE_LIST, + 2 * num_edges); + } + else + { + layout.SetBlockSize(DataLayout::MLD_GRAPH_NODE_LIST, 0); + layout.SetBlockSize(DataLayout::MLD_GRAPH_EDGE_LIST, 0); + } } } @@ -456,6 +497,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // read actual data into shared memory object // // Load the HSGR file + if (boost::filesystem::exists(config.hsgr_data_path)) { io::FileReader hsgr_file(config.hsgr_data_path, io::FileReader::VerifyFingerprint); auto hsgr_header = serialization::readHSGRHeader(hsgr_file); @@ -479,6 +521,14 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) graph_edge_list_ptr, hsgr_header.number_of_edges); } + else + { + layout.GetBlockPtr(memory_ptr, DataLayout::HSGR_CHECKSUM); + layout.GetBlockPtr(memory_ptr, + DataLayout::CH_GRAPH_NODE_LIST); + layout.GetBlockPtr(memory_ptr, + DataLayout::CH_GRAPH_EDGE_LIST); + } // store the filename of the on-disk portion of the RTree { @@ -643,6 +693,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) geometry_input_file.ReadInto(geometries_rev_duration_list_ptr, geometry_node_lists_count); } + if (boost::filesystem::exists(config.datasource_indexes_path)) { io::FileReader geometry_datasource_file(config.datasource_indexes_path, io::FileReader::HasNoFingerprint); @@ -657,9 +708,13 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) geometry_datasource_file, datasources_list_ptr, number_of_compressed_datasources); } } - + else + { + layout.GetBlockPtr(memory_ptr, DataLayout::DATASOURCES_LIST); + } + + if (boost::filesystem::exists(config.datasource_names_path)) { - /* Load names */ io::FileReader datasource_names_file(config.datasource_names_path, io::FileReader::HasNoFingerprint); @@ -706,6 +761,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) datasource_name_lengths_ptr); } } + else + { + layout.GetBlockPtr(memory_ptr, DataLayout::DATASOURCE_NAME_DATA); + layout.GetBlockPtr(memory_ptr, DataLayout::DATASOURCE_NAME_OFFSETS); + layout.GetBlockPtr(memory_ptr, DataLayout::DATASOURCE_NAME_LENGTHS); + } // Loading list of coordinates { @@ -768,6 +829,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) tree_node_file.ReadInto(rtree_ptr, layout.num_entries[DataLayout::R_SEARCH_TREE]); } + if (boost::filesystem::exists(config.core_data_path)) { io::FileReader core_marker_file(config.core_data_path, io::FileReader::HasNoFingerprint); const auto number_of_core_markers = core_marker_file.ReadElementCount32(); @@ -939,6 +1001,40 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) reader.ReadInto(mld_cell_level_offsets_ptr, layout.GetBlockEntries(DataLayout::MLD_CELL_LEVEL_OFFSETS)); } + + if (boost::filesystem::exists(config.edge_based_graph_path)) + { + io::FileReader reader(config.edge_based_graph_path, io::FileReader::VerifyFingerprint); + + const auto number_of_edges = reader.ReadElementCount64(); + const auto number_of_nodes = reader.ReadOne() + 1; + std::vector original_edges(number_of_edges); + reader.ReadInto(original_edges); + + original_edges = partition::SplitBidirectionalEdges(std::move(original_edges)); + auto edges = partition::PrepareEdgesForUsageInGraph(std::move(original_edges)); + BOOST_ASSERT(edges.size() <= 2 * number_of_edges); + + auto nodes_ptr = layout.GetBlockPtr( + memory_ptr, DataLayout::MLD_GRAPH_NODE_LIST); + auto edges_ptr = layout.GetBlockPtr( + memory_ptr, DataLayout::MLD_GRAPH_EDGE_LIST); + + EdgeBasedGraph::EdgeIterator edge = 0; + for (const auto node : util::irange(0u, number_of_nodes + 1)) + { + EdgeBasedGraph::EdgeIterator last_edge = edge; + while (edge < edges.size() && edges[edge].source == node) + { + edges_ptr[edge].target = edges[edge].target; + edges_ptr[edge].data = edges[edge].data; + ++edge; + } + nodes_ptr[node].first_edge = last_edge; + } + + BOOST_ASSERT(edge == edges.size()); + } } } } diff --git a/src/storage/storage_config.cpp b/src/storage/storage_config.cpp index 2127a9ac8..009f5b3fd 100644 --- a/src/storage/storage_config.cpp +++ b/src/storage/storage_config.cpp @@ -7,6 +7,22 @@ namespace osrm { namespace storage { +namespace +{ +bool CheckFileList(const std::vector &files) +{ + bool success = true; + for (auto &path : files) + { + if (!boost::filesystem::is_regular_file(path)) + { + util::Log(logWARNING) << "Missing/Broken File: " << path.string(); + success = false; + } + } + return success; +} +} StorageConfig::StorageConfig(const boost::filesystem::path &base) : ram_index_path{base.string() + ".ramIndex"}, file_index_path{base.string() + ".fileIndex"}, @@ -20,40 +36,40 @@ StorageConfig::StorageConfig(const boost::filesystem::path &base) names_data_path{base.string() + ".names"}, properties_path{base.string() + ".properties"}, intersection_class_path{base.string() + ".icd"}, turn_lane_data_path{base.string() + ".tld"}, turn_lane_description_path{base.string() + ".tls"}, - mld_partition_path{base.string() + ".partition"}, mld_storage_path{base.string() + ".cells"} + mld_partition_path{base.string() + ".partition"}, mld_storage_path{base.string() + ".cells"}, + edge_based_graph_path{base.string() + ".ebg"} { } bool StorageConfig::IsValid() const { const constexpr auto num_files = 15; - const boost::filesystem::path paths[num_files] = {ram_index_path, - file_index_path, - hsgr_data_path, - nodes_data_path, - edges_data_path, - core_data_path, - geometries_path, - timestamp_path, - turn_weight_penalties_path, - turn_duration_penalties_path, - datasource_names_path, - datasource_indexes_path, - names_data_path, - properties_path, - intersection_class_path}; - bool success = true; - for (auto path = paths; path != paths + num_files; ++path) + // Common files + if (!CheckFileList({ram_index_path, + file_index_path, + nodes_data_path, + edges_data_path, + geometries_path, + timestamp_path, + turn_weight_penalties_path, + turn_duration_penalties_path, + names_data_path, + properties_path, + intersection_class_path})) { - if (!boost::filesystem::is_regular_file(*path)) - { - util::Log(logWARNING) << "Missing/Broken File: " << path->string(); - success = false; - } + return false; } - return success; + // TODO: add algorithm checks + + // CH files + CheckFileList({hsgr_data_path, core_data_path, datasource_names_path, datasource_indexes_path}); + + // MLD files + CheckFileList({mld_partition_path, mld_storage_path, edge_based_graph_path}); + + return true; } } } diff --git a/unit_tests/util/static_graph.cpp b/unit_tests/util/static_graph.cpp index 681313c1d..b43b14048 100644 --- a/unit_tests/util/static_graph.cpp +++ b/unit_tests/util/static_graph.cpp @@ -42,7 +42,7 @@ template struct RandomArrayEntryFixture } std::sort(offsets.begin(), offsets.end()); // add sentinel - offsets.push_back(offsets.back()); + offsets.push_back(TEST_NUM_EDGES); // extract interval lengths for (unsigned i = 0; i < offsets.size() - 1; i++)