Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Adds a special graph for MLD with effcient boundary scan

Browse Source 
This graph enables efficient boundary edge scans at each level.
Currenly this needs about |V|*|L| bytes of storage.
We can optimize this when the highest boundary nodes ID is << |V|.
This commit is contained in:
Patrick Niklaus 2017-03-07 03:59:28 +00:00 committed by Patrick Niklaus
parent 58681fa7ea
commit 655ca803d8
12 changed files with 518 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
include/customizer/edge_based_graph.hpp
View File

@ -3,6 +3,7 @@
#include "extractor/edge_based_edge.hpp" #include "extractor/edge_based_edge.hpp"
#include "partition/edge_based_graph.hpp" #include "partition/edge_based_graph.hpp"
#include "partition/multi_level_graph.hpp"
#include "util/static_graph.hpp" #include "util/static_graph.hpp"
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
@ -13,34 +14,24 @@ namespace osrm
namespace customizer namespace customizer
{ {
struct StaticEdgeBasedGraph;
namespace io
{
void read(const boost::filesystem::path &path, StaticEdgeBasedGraph &graph);
void write(const boost::filesystem::path &path, const StaticEdgeBasedGraph &graph);
}
using EdgeBasedGraphEdgeData = partition::EdgeBasedGraphEdgeData; using EdgeBasedGraphEdgeData = partition::EdgeBasedGraphEdgeData;
struct StaticEdgeBasedGraph : util::StaticGraph<EdgeBasedGraphEdgeData> struct MultiLevelEdgeBasedGraph : public partition::MultiLevelGraph<EdgeBasedGraphEdgeData, false>
{ {
using Base = util::StaticGraph<EdgeBasedGraphEdgeData>; using Base = partition::MultiLevelGraph<EdgeBasedGraphEdgeData, false>;
using Base::Base;
friend void io::read(const boost::filesystem::path &path, StaticEdgeBasedGraph &graph);
friend void io::write(const boost::filesystem::path &path, const StaticEdgeBasedGraph &graph);
};
struct StaticEdgeBasedGraphView : util::StaticGraph<EdgeBasedGraphEdgeData, true>
{
using Base = util::StaticGraph<EdgeBasedGraphEdgeData, true>;
using Base::Base; using Base::Base;
}; };
struct StaticEdgeBasedGraphEdge : StaticEdgeBasedGraph::InputEdge struct MultiLevelEdgeBasedGraphView
: public partition::MultiLevelGraph<EdgeBasedGraphEdgeData, true>
{ {
using Base = StaticEdgeBasedGraph::InputEdge; using Base = partition::MultiLevelGraph<EdgeBasedGraphEdgeData, true>;
using Base::Base;
};
struct StaticEdgeBasedGraphEdge : MultiLevelEdgeBasedGraph::InputEdge
{
using Base = MultiLevelEdgeBasedGraph::InputEdge;
using Base::Base; using Base::Base;
}; };
} }

18
include/customizer/io.hpp
View File

@ -11,24 +11,6 @@ namespace customizer
{ {
namespace io namespace io
{ {
inline void read(const boost::filesystem::path &path, StaticEdgeBasedGraph &graph)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(graph.node_array);
reader.DeserializeVector(graph.edge_array);
}
inline void write(const boost::filesystem::path &path, const StaticEdgeBasedGraph &graph)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(graph.node_array);
writer.SerializeVector(graph.edge_array);
}
} }
} }
} }

2
include/engine/datafacade/algorithm_datafacade.hpp
View File

@ -90,6 +90,8 @@ template <> class AlgorithmDataFacade<algorithm::MLD>
virtual const partition::CellStorageView &GetCellStorage() const = 0; virtual const partition::CellStorageView &GetCellStorage() const = 0;
virtual EdgeRange GetBorderEdgeRange(const LevelID level, const NodeID node) const = 0;
// searches for a specific edge // searches for a specific edge
virtual EdgeID FindEdge(const NodeID from, const NodeID to) const = 0; virtual EdgeID FindEdge(const NodeID from, const NodeID to) const = 0;
}; };

131
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View File

@ -902,10 +902,16 @@ class ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>
// MLD data // MLD data
partition::MultiLevelPartitionView mld_partition; partition::MultiLevelPartitionView mld_partition;
partition::CellStorageView mld_cell_storage; partition::CellStorageView mld_cell_storage;
using QueryGraph = customizer::MultiLevelEdgeBasedGraphView;
using GraphNode = QueryGraph::NodeArrayEntry;
using GraphEdge = QueryGraph::EdgeArrayEntry;
QueryGraph query_graph;
void InitializeInternalPointers(storage::DataLayout &data_layout, char *memory_block) void InitializeInternalPointers(storage::DataLayout &data_layout, char *memory_block)
{ {
InitializeMLDDataPointers(data_layout, memory_block); InitializeMLDDataPointers(data_layout, memory_block);
InitializeGraphPointer(data_layout, memory_block);
} }
void InitializeMLDDataPointers(storage::DataLayout &data_layout, char *memory_block) void InitializeMLDDataPointers(storage::DataLayout &data_layout, char *memory_block)
@ -980,6 +986,28 @@ class ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>
std::move(level_offsets)}; std::move(level_offsets)};
} }
} }
void InitializeGraphPointer(storage::DataLayout &data_layout, char *memory_block)
{
auto graph_nodes_ptr = data_layout.GetBlockPtr<GraphNode>(
memory_block, storage::DataLayout::MLD_GRAPH_NODE_LIST);
auto graph_edges_ptr = data_layout.GetBlockPtr<GraphEdge>(
memory_block, storage::DataLayout::MLD_GRAPH_EDGE_LIST);
auto graph_node_to_offset_ptr = data_layout.GetBlockPtr<QueryGraph::EdgeOffset>(
memory_block, storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET);
util::ShM<GraphNode, true>::vector node_list(
graph_nodes_ptr, data_layout.num_entries[storage::DataLayout::MLD_GRAPH_NODE_LIST]);
util::ShM<GraphEdge, true>::vector edge_list(
graph_edges_ptr, data_layout.num_entries[storage::DataLayout::MLD_GRAPH_EDGE_LIST]);
util::ShM<QueryGraph::EdgeOffset, true>::vector node_to_offset(
graph_node_to_offset_ptr,
data_layout.num_entries[storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET]);
query_graph =
QueryGraph(std::move(node_list), std::move(edge_list), std::move(node_to_offset));
}
// allocator that keeps the allocation data // allocator that keeps the allocation data
std::shared_ptr<ContiguousBlockAllocator> allocator; std::shared_ptr<ContiguousBlockAllocator> allocator;
@ -992,86 +1020,63 @@ class ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory()); InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory());
} }
const partition::MultiLevelPartitionView &GetMultiLevelPartition() const const partition::MultiLevelPartitionView &GetMultiLevelPartition() const override
{ {
return mld_partition; return mld_partition;
} }
const partition::CellStorageView &GetCellStorage() const { return mld_cell_storage; } const partition::CellStorageView &GetCellStorage() const override { return mld_cell_storage; }
// search graph access
unsigned GetNumberOfNodes() const override final { return query_graph.GetNumberOfNodes(); }
unsigned GetNumberOfEdges() const override final { return query_graph.GetNumberOfEdges(); }
unsigned GetOutDegree(const NodeID n) const override final
{
return query_graph.GetOutDegree(n);
}
NodeID GetTarget(const EdgeID e) const override final { return query_graph.GetTarget(e); }
const EdgeData &GetEdgeData(const EdgeID e) const override final
{
return query_graph.GetEdgeData(e);
}
EdgeID BeginEdges(const NodeID n) const override final { return query_graph.BeginEdges(n); }
EdgeID EndEdges(const NodeID n) const override final { return query_graph.EndEdges(n); }
EdgeRange GetAdjacentEdgeRange(const NodeID node) const override final
{
return query_graph.GetAdjacentEdgeRange(node);
}
EdgeRange GetBorderEdgeRange(const LevelID level, const NodeID node) const override final
{
return query_graph.GetBorderEdgeRange(level, node);
}
// searches for a specific edge
EdgeID FindEdge(const NodeID from, const NodeID to) const override final
{
return query_graph.FindEdge(from, to);
}
}; };
template <> template <>
class ContiguousInternalMemoryDataFacade<algorithm::MLD> class ContiguousInternalMemoryDataFacade<algorithm::MLD> final
: public ContiguousInternalMemoryDataFacadeBase, : public ContiguousInternalMemoryDataFacadeBase,
public ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD> public ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>
{ {
private: private:
using QueryGraph = customizer::StaticEdgeBasedGraphView;
using GraphNode = QueryGraph::NodeArrayEntry;
using GraphEdge = QueryGraph::EdgeArrayEntry;
std::unique_ptr<QueryGraph> m_query_graph;
void InitializeGraphPointer(storage::DataLayout &data_layout, char *memory_block)
{
auto graph_nodes_ptr = data_layout.GetBlockPtr<GraphNode>(
memory_block, storage::DataLayout::MLD_GRAPH_NODE_LIST);
auto graph_edges_ptr = data_layout.GetBlockPtr<GraphEdge>(
memory_block, storage::DataLayout::MLD_GRAPH_EDGE_LIST);
util::ShM<GraphNode, true>::vector node_list(
graph_nodes_ptr, data_layout.num_entries[storage::DataLayout::MLD_GRAPH_NODE_LIST]);
util::ShM<GraphEdge, true>::vector edge_list(
graph_edges_ptr, data_layout.num_entries[storage::DataLayout::MLD_GRAPH_EDGE_LIST]);
m_query_graph.reset(new QueryGraph(node_list, edge_list));
}
public: public:
ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator) ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator)
: ContiguousInternalMemoryDataFacadeBase(allocator), : ContiguousInternalMemoryDataFacadeBase(allocator),
ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>(allocator) ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>(allocator)
{ {
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory());
}
void InitializeInternalPointers(storage::DataLayout &data_layout, char *memory_block)
{
InitializeGraphPointer(data_layout, memory_block);
}
// search graph access
unsigned GetNumberOfNodes() const override final { return m_query_graph->GetNumberOfNodes(); }
unsigned GetNumberOfEdges() const override final { return m_query_graph->GetNumberOfEdges(); }
unsigned GetOutDegree(const NodeID n) const override final
{
return m_query_graph->GetOutDegree(n);
}
NodeID GetTarget(const EdgeID e) const override final { return m_query_graph->GetTarget(e); }
EdgeData &GetEdgeData(const EdgeID e) const override final
{
return m_query_graph->GetEdgeData(e);
}
EdgeID BeginEdges(const NodeID n) const override final { return m_query_graph->BeginEdges(n); }
EdgeID EndEdges(const NodeID n) const override final { return m_query_graph->EndEdges(n); }
EdgeRange GetAdjacentEdgeRange(const NodeID node) const override final
{
return m_query_graph->GetAdjacentEdgeRange(node);
}
// searches for a specific edge
EdgeID FindEdge(const NodeID from, const NodeID to) const override final
{
return m_query_graph->FindEdge(from, to);
} }
}; };
} }

5
include/engine/routing_algorithms/routing_base_mld.hpp
View File

@ -143,15 +143,14 @@ void routingStep(const datafacade::ContiguousInternalMemoryDataFacade<algorithm:
} }
// Boundary edges // Boundary edges
for (const auto edge : facade.GetAdjacentEdgeRange(node)) for (const auto edge : facade.GetBorderEdgeRange(level, node))
{ {
const auto &edge_data = facade.GetEdgeData(edge); const auto &edge_data = facade.GetEdgeData(edge);
if (DIRECTION == FORWARD_DIRECTION ? edge_data.forward : edge_data.backward) if (DIRECTION == FORWARD_DIRECTION ? edge_data.forward : edge_data.backward)
{ {
const NodeID to = facade.GetTarget(edge); const NodeID to = facade.GetTarget(edge);
if (checkParentCellRestriction(partition.GetCell(level + 1, to), args...) && if (checkParentCellRestriction(partition.GetCell(level + 1, to), args...))
partition.GetHighestDifferentLevel(node, to) >= level)
{ {
BOOST_ASSERT_MSG(edge_data.weight > 0, "edge_weight invalid"); BOOST_ASSERT_MSG(edge_data.weight > 0, "edge_weight invalid");
const EdgeWeight to_weight = weight + edge_data.weight; const EdgeWeight to_weight = weight + edge_data.weight;

25
include/partition/io.hpp
View File

@ -3,6 +3,7 @@
#include "partition/cell_storage.hpp" #include "partition/cell_storage.hpp"
#include "partition/edge_based_graph.hpp" #include "partition/edge_based_graph.hpp"
#include "partition/multi_level_graph.hpp"
#include "partition/multi_level_partition.hpp" #include "partition/multi_level_partition.hpp"
#include "storage/io.hpp" #include "storage/io.hpp"
@ -14,6 +15,30 @@ namespace partition
namespace io namespace io
{ {
template <typename EdgeDataT, bool UseSharedMemory>
inline void read(const boost::filesystem::path &path,
MultiLevelGraph<EdgeDataT, UseSharedMemory> &graph)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(graph.node_array);
reader.DeserializeVector(graph.edge_array);
reader.DeserializeVector(graph.edge_to_level);
}
template <typename EdgeDataT, bool UseSharedMemory>
inline void write(const boost::filesystem::path &path,
const MultiLevelGraph<EdgeDataT, UseSharedMemory> &graph)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(graph.node_array);
writer.SerializeVector(graph.edge_array);
writer.SerializeVector(graph.node_to_edge_offset);
}
template <> inline void read(const boost::filesystem::path &path, MultiLevelPartition &mlp) template <> inline void read(const boost::filesystem::path &path, MultiLevelPartition &mlp)
{ {
const auto fingerprint = storage::io::FileReader::VerifyFingerprint; const auto fingerprint = storage::io::FileReader::VerifyFingerprint;

191
include/partition/multi_level_graph.hpp Normal file
View File

@ -0,0 +1,191 @@
#ifndef OSRM_PARTITION_MULTI_LEVEL_GRAPH_HPP
#define OSRM_PARTITION_MULTI_LEVEL_GRAPH_HPP
#include "partition/multi_level_partition.hpp"
#include "util/static_graph.hpp"
#include <tbb/parallel_sort.h>
#include <boost/iterator/permutation_iterator.hpp>
#include <boost/range/combine.hpp>
namespace osrm
{
namespace partition
{
template <typename EdgeDataT, bool UseSharedMemory> class MultiLevelGraph;
namespace io
{
template <typename EdgeDataT, bool UseSharedMemory>
void read(const boost::filesystem::path &path, MultiLevelGraph<EdgeDataT, UseSharedMemory> &graph);
template <typename EdgeDataT, bool UseSharedMemory>
void write(const boost::filesystem::path &path,
const MultiLevelGraph<EdgeDataT, UseSharedMemory> &graph);
}
template <typename EdgeDataT, bool UseSharedMemory>
class MultiLevelGraph : public util::StaticGraph<EdgeDataT, UseSharedMemory>
{
private:
using SuperT = util::StaticGraph<EdgeDataT, UseSharedMemory>;
template <typename T> using Vector = typename util::ShM<T, UseSharedMemory>::vector;
public:
// We limit each node to have 255 edges
// this is very generous, we could probably pack this
using EdgeOffset = std::uint8_t;
MultiLevelGraph() = default;
MultiLevelGraph(Vector<typename SuperT::NodeArrayEntry> node_array_,
Vector<typename SuperT::EdgeArrayEntry> edge_array_,
Vector<EdgeOffset> node_to_edge_offset_)
: SuperT(std::move(node_array_), std::move(edge_array_)),
node_to_edge_offset(std::move(node_to_edge_offset_))
{
}
template <typename ContainerT>
MultiLevelGraph(const MultiLevelPartition &mlp,
const std::uint32_t num_nodes,
const ContainerT &edges)
{
auto highest_border_level = GetHighestBorderLevel(mlp, edges);
auto permutation = SortEdgesByHighestLevel(highest_border_level, edges);
auto sorted_edges_begin =
boost::make_permutation_iterator(edges.begin(), permutation.begin());
auto sorted_edges_end = boost::make_permutation_iterator(edges.begin(), permutation.end());
SuperT::InitializeFromSortedEdgeRange(num_nodes, sorted_edges_begin, sorted_edges_end);
// if the node ordering is sorting the border nodes first,
// the id of the maximum border node will be rather low
// enabling us to save some memory here
auto max_border_node_id = 0u;
for (auto edge_index : util::irange<std::size_t>(0, edges.size()))
{
if (highest_border_level[edge_index] > 0)
{
max_border_node_id =
std::max(max_border_node_id,
std::max(edges[edge_index].source, edges[edge_index].target));
}
}
BOOST_ASSERT(max_border_node_id < num_nodes);
auto edge_and_level_range = boost::combine(edges, highest_border_level);
auto sorted_edge_and_level_begin =
boost::make_permutation_iterator(edge_and_level_range.begin(), permutation.begin());
auto sorted_edge_and_level_end =
boost::make_permutation_iterator(edge_and_level_range.begin(), permutation.end());
InitializeOffsetsFromSortedEdges(
mlp, max_border_node_id, sorted_edge_and_level_begin, sorted_edge_and_level_end);
}
// Fast scan over all relevant border edges
auto GetBorderEdgeRange(const LevelID level, const NodeID node) const
{
auto begin = BeginBorderEdges(level, node);
auto end = SuperT::EndEdges(node);
return util::irange<EdgeID>(begin, end);
}
// Fast scan over all relevant internal edges, that is edges that will not
// leave the cell of that node at the given level
auto GetInternalEdgeRange(const LevelID level, const NodeID node) const
{
auto begin = SuperT::BeginEdges(node);
auto end = SuperT::BeginEdges(node) + node_to_edge_offset[node + level];
return util::irange<EdgeID>(begin, end);
}
EdgeID BeginBorderEdges(const LevelID level, const NodeID node) const
{
auto index = node * GetNumberOfLevels();
if (index >= node_to_edge_offset.size() - 1)
return SuperT::BeginEdges(node);
else
return SuperT::BeginEdges(node) + node_to_edge_offset[index + level];
}
// We save the level as senitel at the end
LevelID GetNumberOfLevels() const { return node_to_edge_offset.back(); }
private:
template <typename ContainerT>
auto GetHighestBorderLevel(const MultiLevelPartition &mlp, const ContainerT &edges) const
{
std::vector<LevelID> highest_border_level(edges.size());
std::transform(
edges.begin(), edges.end(), highest_border_level.begin(), [&mlp](const auto &edge) {
return mlp.GetHighestDifferentLevel(edge.source, edge.target);
});
return highest_border_level;
}
template <typename ContainerT>
auto SortEdgesByHighestLevel(const std::vector<LevelID> &highest_border_level,
const ContainerT &edges) const
{
std::vector<std::uint32_t> permutation(edges.size());
std::iota(permutation.begin(), permutation.end(), 0);
tbb::parallel_sort(
permutation.begin(),
permutation.end(),
[&edges, &highest_border_level](const auto &lhs, const auto &rhs) {
// sort by source node and then by level in acending order
return std::tie(edges[lhs].source, highest_border_level[lhs], edges[lhs].target) <
std::tie(edges[rhs].source, highest_border_level[rhs], edges[rhs].target);
});
return permutation;
}
template <typename ZipIterT>
auto InitializeOffsetsFromSortedEdges(const MultiLevelPartition &mlp,
const NodeID max_border_node_id,
ZipIterT edge_and_level_begin,
ZipIterT edge_and_level_end)
{
auto num_levels = mlp.GetNumberOfLevels();
// we save one senitel element at the end
node_to_edge_offset.reserve(num_levels * (max_border_node_id + 1) + 1);
auto iter = edge_and_level_begin;
for (auto node : util::irange<NodeID>(0, max_border_node_id + 1))
{
node_to_edge_offset.push_back(0);
auto level_begin = iter;
for (auto level : util::irange<LevelID>(0, mlp.GetNumberOfLevels()))
{
iter = std::find_if(
iter, edge_and_level_end, [node, level](const auto &edge_and_level) {
return boost::get<0>(edge_and_level).source != node ||
boost::get<1>(edge_and_level) != level;
});
EdgeOffset offset = std::distance(level_begin, iter);
node_to_edge_offset.push_back(offset);
}
node_to_edge_offset.pop_back();
}
BOOST_ASSERT(node_to_edge_offset.size() ==
mlp.GetNumberOfLevels() * (max_border_node_id + 1));
// save number of levels as last element so we can reconstruct the stride
node_to_edge_offset.push_back(mlp.GetNumberOfLevels());
}
friend void
io::read<EdgeDataT, UseSharedMemory>(const boost::filesystem::path &path,
MultiLevelGraph<EdgeDataT, UseSharedMemory> &graph);
friend void
io::write<EdgeDataT, UseSharedMemory>(const boost::filesystem::path &path,
const MultiLevelGraph<EdgeDataT, UseSharedMemory> &graph);
Vector<EdgeOffset> node_to_edge_offset;
};
}
}
#endif

4
include/storage/shared_datatype.hpp
View File

@ -64,7 +64,8 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
"MLD_CELLS", "MLD_CELLS",
"MLD_CELL_LEVEL_OFFSETS", "MLD_CELL_LEVEL_OFFSETS",
"MLD_GRAPH_NODE_LIST", "MLD_GRAPH_NODE_LIST",
"MLD_GRAPH_EDGE_LIST"}; "MLD_GRAPH_EDGE_LIST",
"MLD_GRAPH_NODE_TO_OFFSET"};
struct DataLayout struct DataLayout
{ {
@ -117,6 +118,7 @@ struct DataLayout
MLD_CELL_LEVEL_OFFSETS, MLD_CELL_LEVEL_OFFSETS,
MLD_GRAPH_NODE_LIST, MLD_GRAPH_NODE_LIST,
MLD_GRAPH_EDGE_LIST, MLD_GRAPH_EDGE_LIST,
MLD_GRAPH_NODE_TO_OFFSET,
NUM_BLOCKS NUM_BLOCKS
}; };

21
include/util/static_graph.hpp
View File

@ -112,6 +112,8 @@ template <typename EdgeDataT, bool UseSharedMemory = false> class StaticGraph
return irange(BeginEdges(node), EndEdges(node)); return irange(BeginEdges(node), EndEdges(node));
} }
StaticGraph() {}
template <typename ContainerT> StaticGraph(const int nodes, const ContainerT &edges) template <typename ContainerT> StaticGraph(const int nodes, const ContainerT &edges)
{ {
BOOST_ASSERT(std::is_sorted(const_cast<ContainerT &>(edges).begin(), BOOST_ASSERT(std::is_sorted(const_cast<ContainerT &>(edges).begin(),
@ -120,18 +122,15 @@ template <typename EdgeDataT, bool UseSharedMemory = false> class StaticGraph
InitializeFromSortedEdgeRange(nodes, edges.begin(), edges.end()); InitializeFromSortedEdgeRange(nodes, edges.begin(), edges.end());
} }
StaticGraph(typename ShM<NodeArrayEntry, UseSharedMemory>::vector &nodes, StaticGraph(typename ShM<NodeArrayEntry, UseSharedMemory>::vector node_array_,
typename ShM<EdgeArrayEntry, UseSharedMemory>::vector &edges) typename ShM<EdgeArrayEntry, UseSharedMemory>::vector edge_array_)
: node_array(std::move(node_array_)), edge_array(std::move(edge_array_))
{ {
BOOST_ASSERT(!nodes.empty()); BOOST_ASSERT(!node_array.empty());
number_of_nodes = static_cast<decltype(number_of_nodes)>(nodes.size() - 1); number_of_nodes = static_cast<decltype(number_of_nodes)>(node_array.size() - 1);
number_of_edges = static_cast<decltype(number_of_edges)>(nodes.back().first_edge); number_of_edges = static_cast<decltype(number_of_edges)>(node_array.back().first_edge);
BOOST_ASSERT(number_of_edges <= edges.size()); BOOST_ASSERT(number_of_edges <= edge_array.size());
using std::swap;
swap(node_array, nodes);
swap(edge_array, edges);
} }
unsigned GetNumberOfNodes() const { return number_of_nodes; } unsigned GetNumberOfNodes() const { return number_of_nodes; }
@ -255,7 +254,7 @@ template <typename EdgeDataT, bool UseSharedMemory = false> class StaticGraph
}); });
} }
private: // private:
NodeIterator number_of_nodes; NodeIterator number_of_nodes;
EdgeIterator number_of_edges; EdgeIterator number_of_edges;

14
src/customize/customizer.cpp
View File

@ -6,7 +6,6 @@
#include "partition/cell_storage.hpp" #include "partition/cell_storage.hpp"
#include "partition/edge_based_graph_reader.hpp" #include "partition/edge_based_graph_reader.hpp"
#include "partition/io.hpp" #include "partition/io.hpp"
#include "partition/io.hpp"
#include "partition/multi_level_partition.hpp" #include "partition/multi_level_partition.hpp"
#include "updater/updater.hpp" #include "updater/updater.hpp"
@ -71,7 +70,8 @@ void CellStorageStatistics(const Graph &graph,
} }
} }
auto LoadAndUpdateEdgeExpandedGraph(const CustomizationConfig &config) auto LoadAndUpdateEdgeExpandedGraph(const CustomizationConfig &config,
const partition::MultiLevelPartition &mlp)
{ {
updater::Updater updater(config.updater_config); updater::Updater updater(config.updater_config);
@ -82,7 +82,9 @@ auto LoadAndUpdateEdgeExpandedGraph(const CustomizationConfig &config)
auto directed = partition::splitBidirectionalEdges(edge_based_edge_list); auto directed = partition::splitBidirectionalEdges(edge_based_edge_list);
auto tidied = auto tidied =
partition::prepareEdgesForUsageInGraph<StaticEdgeBasedGraphEdge>(std::move(directed)); partition::prepareEdgesForUsageInGraph<StaticEdgeBasedGraphEdge>(std::move(directed));
auto edge_based_graph = std::make_unique<StaticEdgeBasedGraph>(num_nodes, std::move(tidied)); auto edge_based_graph =
std::make_unique<partition::MultiLevelGraph<EdgeBasedGraphEdgeData, false>>(
mlp, num_nodes, std::move(tidied));
util::Log() << "Loaded edge based graph for mapping partition ids: " util::Log() << "Loaded edge based graph for mapping partition ids: "
<< edge_based_graph->GetNumberOfEdges() << " edges, " << edge_based_graph->GetNumberOfEdges() << " edges, "
@ -95,11 +97,11 @@ int Customizer::Run(const CustomizationConfig &config)
{ {
TIMER_START(loading_data); TIMER_START(loading_data);
auto edge_based_graph = LoadAndUpdateEdgeExpandedGraph(config);
partition::MultiLevelPartition mlp; partition::MultiLevelPartition mlp;
partition::io::read(config.mld_partition_path, mlp); partition::io::read(config.mld_partition_path, mlp);
auto edge_based_graph = LoadAndUpdateEdgeExpandedGraph(config, mlp);
partition::CellStorage storage; partition::CellStorage storage;
partition::io::read(config.mld_storage_path, storage); partition::io::read(config.mld_storage_path, storage);
TIMER_STOP(loading_data); TIMER_STOP(loading_data);
@ -117,7 +119,7 @@ int Customizer::Run(const CustomizationConfig &config)
util::Log() << "MLD customization writing took " << TIMER_SEC(writing_mld_data) << " seconds"; util::Log() << "MLD customization writing took " << TIMER_SEC(writing_mld_data) << " seconds";
TIMER_START(writing_graph); TIMER_START(writing_graph);
io::write(config.mld_graph_path, *edge_based_graph); partition::io::write(config.mld_graph_path, *edge_based_graph);
TIMER_STOP(writing_graph); TIMER_STOP(writing_graph);
util::Log() << "Graph writing took " << TIMER_SEC(writing_graph) << " seconds"; util::Log() << "Graph writing took " << TIMER_SEC(writing_graph) << " seconds";

27
src/storage/storage.cpp
View File

@ -447,21 +447,27 @@ void Storage::PopulateLayout(DataLayout &layout)
io::FileReader reader(config.mld_graph_path, io::FileReader::VerifyFingerprint); io::FileReader reader(config.mld_graph_path, io::FileReader::VerifyFingerprint);
const auto num_nodes = const auto num_nodes =
reader.ReadVectorSize<customizer::StaticEdgeBasedGraph::NodeArrayEntry>(); reader.ReadVectorSize<customizer::MultiLevelEdgeBasedGraph::NodeArrayEntry>();
const auto num_edges = const auto num_edges =
reader.ReadVectorSize<customizer::StaticEdgeBasedGraph::EdgeArrayEntry>(); reader.ReadVectorSize<customizer::MultiLevelEdgeBasedGraph::EdgeArrayEntry>();
const auto num_node_offsets =
reader.ReadVectorSize<customizer::MultiLevelEdgeBasedGraph::EdgeOffset>();
layout.SetBlockSize<customizer::StaticEdgeBasedGraph::NodeArrayEntry>( layout.SetBlockSize<customizer::MultiLevelEdgeBasedGraph::NodeArrayEntry>(
DataLayout::MLD_GRAPH_NODE_LIST, num_nodes); DataLayout::MLD_GRAPH_NODE_LIST, num_nodes);
layout.SetBlockSize<customizer::StaticEdgeBasedGraph::EdgeArrayEntry>( layout.SetBlockSize<customizer::MultiLevelEdgeBasedGraph::EdgeArrayEntry>(
DataLayout::MLD_GRAPH_EDGE_LIST, num_edges); DataLayout::MLD_GRAPH_EDGE_LIST, num_edges);
layout.SetBlockSize<customizer::MultiLevelEdgeBasedGraph::EdgeOffset>(
DataLayout::MLD_GRAPH_NODE_TO_OFFSET, num_node_offsets);
} }
else else
{ {
layout.SetBlockSize<customizer::StaticEdgeBasedGraph::NodeArrayEntry>( layout.SetBlockSize<customizer::MultiLevelEdgeBasedGraph::NodeArrayEntry>(
DataLayout::MLD_GRAPH_NODE_LIST, 0); DataLayout::MLD_GRAPH_NODE_LIST, 0);
layout.SetBlockSize<customizer::StaticEdgeBasedGraph::EdgeArrayEntry>( layout.SetBlockSize<customizer::MultiLevelEdgeBasedGraph::EdgeArrayEntry>(
DataLayout::MLD_GRAPH_EDGE_LIST, 0); DataLayout::MLD_GRAPH_EDGE_LIST, 0);
layout.SetBlockSize<customizer::MultiLevelEdgeBasedGraph::EdgeOffset>(
DataLayout::MLD_GRAPH_NODE_TO_OFFSET, 0);
} }
} }
} }
@ -938,16 +944,21 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
io::FileReader reader(config.mld_graph_path, io::FileReader::VerifyFingerprint); io::FileReader reader(config.mld_graph_path, io::FileReader::VerifyFingerprint);
auto nodes_ptr = auto nodes_ptr =
layout.GetBlockPtr<customizer::StaticEdgeBasedGraph::NodeArrayEntry, true>( layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraph::NodeArrayEntry, true>(
memory_ptr, DataLayout::MLD_GRAPH_NODE_LIST); memory_ptr, DataLayout::MLD_GRAPH_NODE_LIST);
auto edges_ptr = auto edges_ptr =
layout.GetBlockPtr<customizer::StaticEdgeBasedGraph::EdgeArrayEntry, true>( layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraph::EdgeArrayEntry, true>(
memory_ptr, DataLayout::MLD_GRAPH_EDGE_LIST); memory_ptr, DataLayout::MLD_GRAPH_EDGE_LIST);
auto node_to_offset_ptr =
layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraph::EdgeOffset, true>(
memory_ptr, DataLayout::MLD_GRAPH_NODE_TO_OFFSET);
auto num_nodes = reader.ReadElementCount64(); auto num_nodes = reader.ReadElementCount64();
reader.ReadInto(nodes_ptr, num_nodes); reader.ReadInto(nodes_ptr, num_nodes);
auto num_edges = reader.ReadElementCount64(); auto num_edges = reader.ReadElementCount64();
reader.ReadInto(edges_ptr, num_edges); reader.ReadInto(edges_ptr, num_edges);
auto num_node_to_offset = reader.ReadElementCount64();
reader.ReadInto(node_to_offset_ptr, num_node_to_offset);
} }
} }
} }

178
unit_tests/partition/multi_level_graph.cpp Normal file
View File

@ -0,0 +1,178 @@
#include "partition/multi_level_graph.hpp"
#include "util/typedefs.hpp"
#include "../common/range_tools.hpp"
#include <boost/test/test_case_template.hpp>
#include <boost/test/unit_test.hpp>
#include <algorithm>
#include <random>
#include <vector>
using namespace osrm;
using namespace osrm::partition;
namespace
{
struct MockEdge
{
NodeID source;
NodeID target;
};
auto makeGraph(const MultiLevelPartition &mlp, const std::vector<MockEdge> &mock_edges)
{
struct EdgeData
{
bool forward;
bool backward;
};
using Edge = util::static_graph_details::SortableEdgeWithData<EdgeData>;
std::vector<Edge> edges;
std::size_t max_id = 0;
for (const auto &m : mock_edges)
{
max_id = std::max<std::size_t>(max_id, std::max(m.source, m.target));
edges.push_back(Edge{m.source, m.target, true, false});
edges.push_back(Edge{m.target, m.source, false, true});
}
std::sort(edges.begin(), edges.end());
return MultiLevelGraph<EdgeData, false>(mlp, max_id + 1, edges);
}
}
BOOST_AUTO_TEST_SUITE(multi_level_graph)
BOOST_AUTO_TEST_CASE(check_edges_sorting)
{
// node: 0 1 2 3 4 5 6 7 8 9 10 11 12
std::vector<CellID> l1{{0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6}};
std::vector<CellID> l2{{0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4}};
std::vector<CellID> l3{{0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2}};
std::vector<CellID> l4{{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}};
MultiLevelPartition mlp{{l1, l2, l3, l4}, {7, 5, 3, 2}};
std::vector<MockEdge> edges = {
// edges sorted into border/internal by level
// level: (1) (2) (3) (4)
{0, 1}, // i i i i
{2, 3}, // i i i i
{3, 7}, // b b b i
{4, 0}, // b b b i
{4, 5}, // i i i i
{5, 6}, // b i i i
{6, 4}, // b i i i
{6, 7}, // i i i i
{7, 11}, // b b i i
{8, 9}, // i i i i
{9, 8}, // i i i i
{10, 11}, // i i i i
{11, 10}, // i i i i
{11, 12} // b b b b
};
auto graph = makeGraph(mlp, edges);
for (auto from : util::irange(0u, graph.GetNumberOfNodes()))
{
LevelID level = 0;
for (auto edge : graph.GetAdjacentEdgeRange(from))
{
auto to = graph.GetTarget(edge);
BOOST_CHECK(mlp.GetHighestDifferentLevel(from, to) >= level);
level = mlp.GetHighestDifferentLevel(from, to);
}
}
// on level 0 every edge is a border edge
for (auto node : util::irange<NodeID>(0, 13))
CHECK_EQUAL_COLLECTIONS(graph.GetBorderEdgeRange(0, node),
graph.GetAdjacentEdgeRange(node));
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(1, 0).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(1, 1).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(1, 2).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(1, 3).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(1, 4).size(), 2);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(1, 5).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(1, 6).size(), 2);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(1, 7).size(), 2);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(1, 8).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(1, 9).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(1, 10).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(1, 11).size(), 2);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(1, 12).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(2, 0).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(2, 1).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(2, 2).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(2, 3).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(2, 4).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(2, 5).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(2, 6).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(2, 7).size(), 2);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(2, 8).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(2, 9).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(2, 10).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(2, 11).size(), 2);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(2, 12).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(3, 0).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(3, 1).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(3, 2).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(3, 3).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(3, 4).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(3, 5).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(3, 6).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(3, 7).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(3, 8).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(3, 9).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(3, 10).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(3, 11).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(3, 12).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(4, 0).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(4, 1).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(4, 2).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(4, 3).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(4, 4).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(4, 5).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(4, 6).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(4, 7).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(4, 8).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(4, 9).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(4, 10).size(), 0);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(4, 11).size(), 1);
BOOST_CHECK_EQUAL(graph.GetBorderEdgeRange(3, 12).size(), 1);
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(1, 0), graph.FindEdge(0, 4));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(1, 3), graph.FindEdge(3, 7));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(1, 4), graph.FindEdge(4, 6), graph.FindEdge(4, 0));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(1, 5), graph.FindEdge(5, 6));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(1, 6), graph.FindEdge(6, 4), graph.FindEdge(6, 5));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(1, 7), graph.FindEdge(7, 11), graph.FindEdge(7, 3));
CHECK_EQUAL_RANGE(
graph.GetBorderEdgeRange(1, 11), graph.FindEdge(11, 7), graph.FindEdge(11, 12));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(1, 12), graph.FindEdge(12, 11));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(2, 0), graph.FindEdge(0, 4));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(2, 3), graph.FindEdge(3, 7));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(2, 4), graph.FindEdge(4, 0));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(2, 7), graph.FindEdge(7, 11), graph.FindEdge(7, 3));
CHECK_EQUAL_RANGE(
graph.GetBorderEdgeRange(2, 11), graph.FindEdge(11, 7), graph.FindEdge(11, 12));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(2, 12), graph.FindEdge(12, 11));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(3, 0), graph.FindEdge(0, 4));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(3, 3), graph.FindEdge(3, 7));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(3, 4), graph.FindEdge(4, 0));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(3, 7), graph.FindEdge(7, 3));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(3, 11), graph.FindEdge(11, 12));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(3, 12), graph.FindEdge(12, 11));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(4, 11), graph.FindEdge(11, 12));
CHECK_EQUAL_RANGE(graph.GetBorderEdgeRange(4, 12), graph.FindEdge(12, 11));
}
BOOST_AUTO_TEST_SUITE_END()