Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Refactor metric storage

Browse Source
This commit is contained in:
Patrick Niklaus
2018-03-27 09:44:13 +00:00
committed by Patrick Niklaus
parent aec9b6a178
commit c334d11e95
14 changed files with 505 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/engine/algorithm.hpp
+5
View File
@@ -30,6 +30,11 @@ template <typename AlgorithmT> const char *name();
template <> inline const char *name<ch::Algorithm>() { return "CH"; }
template <> inline const char *name<mld::Algorithm>() { return "MLD"; }
// Algorithm identifier
template <typename AlgorithmT> const char *identifier();
template <> inline const char *identifier<ch::Algorithm>() { return "ch"; }
template <> inline const char *identifier<mld::Algorithm>() { return "mld"; }
template <typename AlgorithmT> struct HasAlternativePathSearch final : std::false_type
{
};
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+35 -22
View File
@@ -83,23 +83,26 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
void InitializeGraphPointer(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
const std::string metric_prefix = "/ch/metrics/" + metric_name;
auto graph_nodes_ptr =
data_layout.GetBlockPtr<GraphNode>(memory_block, "/ch/contracted_graph/node_array");
data_layout.GetBlockPtr<GraphNode>(memory_block, metric_prefix + "/contracted_graph/node_array");
auto graph_edges_ptr =
data_layout.GetBlockPtr<GraphEdge>(memory_block, "/ch/contracted_graph/edge_array");
data_layout.GetBlockPtr<GraphEdge>(memory_block, metric_prefix + "/contracted_graph/edge_array");
auto filter_block_id = "/ch/edge_filter/" + std::to_string(exclude_index);
auto exclude_prefix = metric_prefix + "/exclude/" + std::to_string(exclude_index);
auto filter_block_id = exclude_prefix + "/edge_filter";
auto edge_filter_ptr =
data_layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_block, filter_block_id);
util::vector_view<GraphNode> node_list(
graph_nodes_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/node_array"));
graph_nodes_ptr, data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/node_array"));
util::vector_view<GraphEdge> edge_list(
graph_edges_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/edge_array"));
graph_edges_ptr, data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/edge_array"));
util::vector_view<bool> edge_filter(edge_filter_ptr,
data_layout.GetBlockEntries(filter_block_id));
@@ -108,17 +111,18 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
public:
ContiguousInternalMemoryAlgorithmDataFacade(
std::shared_ptr<ContiguousBlockAllocator> allocator_, std::size_t exclude_index)
std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::string& metric_name, std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
}
void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
InitializeGraphPointer(data_layout, memory_block, exclude_index);
InitializeGraphPointer(data_layout, memory_block, metric_name, exclude_index);
}
// search graph access
@@ -225,8 +229,11 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
void InitializeProfilePropertiesPointer(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
// TODO: For multi-metric support we need to have separate exclude classes per metric
(void) metric_name;
m_profile_properties = data_layout.GetBlockPtr<extractor::ProfileProperties>(
memory_block, "/common/properties");
@@ -528,6 +535,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
InitializeChecksumPointer(data_layout, memory_block);
@@ -538,7 +546,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
InitializeGeometryPointers(data_layout, memory_block);
InitializeNamePointers(data_layout, memory_block);
InitializeTurnLaneDescriptionsPointers(data_layout, memory_block);
InitializeProfilePropertiesPointer(data_layout, memory_block, exclude_index);
InitializeProfilePropertiesPointer(data_layout, memory_block, metric_name, exclude_index);
InitializeRTreePointers(data_layout, memory_block);
InitializeIntersectionClassPointers(data_layout, memory_block);
InitializeManeuverOverridePointers(data_layout, memory_block);
@@ -548,10 +556,11 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
// allows switching between process_memory/shared_memory datafacade, based on the type of
// allocator
ContiguousInternalMemoryDataFacadeBase(std::shared_ptr<ContiguousBlockAllocator> allocator_,
const std::string& metric_name,
const std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
}
// node and edge information access
@@ -959,9 +968,10 @@ class ContiguousInternalMemoryDataFacade<CH>
{
public:
ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator,
const std::string& metric_name,
const std::size_t exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<CH>(allocator, exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<CH>(allocator, metric_name, exclude_index)
{
}
@@ -981,14 +991,16 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
InitializeMLDDataPointers(data_layout, memory_block, exclude_index);
InitializeMLDDataPointers(data_layout, memory_block, metric_name, exclude_index);
InitializeGraphPointer(data_layout, memory_block);
}
void InitializeMLDDataPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
if (data_layout.GetBlockSize("/mld/multilevelpartition/partition") > 0)
@@ -1016,9 +1028,9 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
partitioner::MultiLevelPartitionView{level_data, partition, cell_to_children};
}
const auto weights_block_id = "/mld/metrics/" + std::to_string(exclude_index) + "/weights";
const auto durations_block_id =
"/mld/metrics/" + std::to_string(exclude_index) + "/durations";
const auto exclude_prefix = "/mld/metrics/" + metric_name + "/exclude/" + std::to_string(exclude_index);
const auto weights_block_id = exclude_prefix + "/weights";
const auto durations_block_id = exclude_prefix + "/durations";
if (data_layout.GetBlockSize(weights_block_id) > 0)
{
@@ -1026,8 +1038,8 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
data_layout.GetBlockPtr<EdgeWeight>(memory_block, weights_block_id);
auto mld_cell_durations_ptr =
data_layout.GetBlockPtr<EdgeDuration>(memory_block, durations_block_id);
auto weight_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/weights");
auto duration_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/durations");
auto weight_entries_count = data_layout.GetBlockEntries(weights_block_id);
auto duration_entries_count = data_layout.GetBlockEntries(durations_block_id);
BOOST_ASSERT(weight_entries_count == duration_entries_count);
util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
util::vector_view<EdgeDuration> durations(mld_cell_durations_ptr,
@@ -1099,10 +1111,10 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
public:
ContiguousInternalMemoryAlgorithmDataFacade(
std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::size_t exclude_index)
std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::string& metric_name, const std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
}
const partitioner::MultiLevelPartitionView &GetMultiLevelPartition() const override
@@ -1156,9 +1168,10 @@ class ContiguousInternalMemoryDataFacade<MLD> final
private:
public:
ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator,
const std::string& metric_name,
const std::size_t exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<MLD>(allocator, exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<MLD>(allocator, metric_name, exclude_index)
{
}
include/engine/datafacade_factory.hpp
+27 -8
View File
@@ -33,6 +33,7 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
DataFacadeFactory(std::shared_ptr<AllocatorT> allocator)
: DataFacadeFactory(allocator, has_exclude_flags)
{
BOOST_ASSERT_MSG(facades.size() >= 1, "At least one datafacade is needed");
}
template <typename ParameterT> std::shared_ptr<const Facade> Get(const ParameterT &params) const
@@ -45,13 +46,27 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
template <typename AllocatorT>
DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::true_type)
{
for (const auto index : util::irange<std::size_t>(0, facades.size()))
{
facades[index] = std::make_shared<const Facade>(allocator, index);
}
properties = allocator->GetLayout().template GetBlockPtr<extractor::ProfileProperties>(
const auto &layout = allocator->GetLayout();
properties = layout.template GetBlockPtr<extractor::ProfileProperties>(
allocator->GetMemory(), "/common/properties");
const auto &metric_name = properties->GetWeightName();
std::vector<std::string> exclude_prefixes;
auto exclude_path = std::string("/") + routing_algorithms::identifier<AlgorithmT>() +
std::string("/metrics/") + metric_name + "/exclude/";
layout.List(exclude_path, std::back_inserter(exclude_prefixes));
facades.resize(exclude_prefixes.size());
for (const auto &exclude_prefix : exclude_prefixes)
{
auto index_begin = exclude_prefix.find_last_of("/");
BOOST_ASSERT_MSG(index_begin != std::string::npos,
"The exclude prefix needs to be a valid data path.");
std::size_t index =
std::stoi(exclude_prefix.substr(index_begin + 1, exclude_prefix.size()));
BOOST_ASSERT(index >= 0 && index < facades.size());
facades[index] = std::make_shared<const Facade>(allocator, metric_name, index);
}
for (const auto index : util::irange<std::size_t>(0, properties->class_names.size()))
{
@@ -67,7 +82,11 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
template <typename AllocatorT>
DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::false_type)
{
facades[0] = std::make_shared<const Facade>(allocator, 0);
const auto &layout = allocator->GetLayout();
properties = layout.template GetBlockPtr<extractor::ProfileProperties>(
allocator->GetMemory(), "/common/properties");
const auto &metric_name = properties->GetWeightName();
facades.push_back(std::make_shared<const Facade>(allocator, metric_name, 0));
}
std::shared_ptr<const Facade> Get(const api::TileParameters &, std::false_type) const
@@ -124,7 +143,7 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
return {};
}
std::array<std::shared_ptr<const Facade>, extractor::MAX_EXCLUDABLE_CLASSES> facades;
std::vector<std::shared_ptr<const Facade>> facades;
std::unordered_map<std::string, extractor::ClassData> name_to_class;
const extractor::ProfileProperties *properties = nullptr;
};
include/engine/engine.hpp
+32 -15
View File
@@ -155,10 +155,19 @@ bool Engine<routing_algorithms::ch::Algorithm>::CheckCompatibility(const EngineC
auto mem = storage::makeSharedMemory(barrier.data().region);
storage::DataLayout layout;
storage::io::BufferReader reader(reinterpret_cast<const char*>(mem->Ptr()), mem->Size());
storage::io::BufferReader reader(reinterpret_cast<const char *>(mem->Ptr()), mem->Size());
storage::serialization::read(reader, layout);
return layout.HasBlock("/ch/contracted_graph/node_array") &&
layout.HasBlock("/ch/contracted_graph/edge_array");
std::vector<std::string> metric_prefixes;
layout.List("/ch/metrics/", std::back_inserter(metric_prefixes));
bool has_graph = false;
for (const auto &metric_prefix : metric_prefixes)
{
has_graph |= layout.HasBlock(metric_prefix + "/contracted_graph/node_array") &&
layout.HasBlock(metric_prefix + "/contracted_graph/edge_array");
}
return has_graph;
}
else
{
@@ -177,24 +186,32 @@ bool Engine<routing_algorithms::mld::Algorithm>::CheckCompatibility(const Engine
auto mem = storage::makeSharedMemory(barrier.data().region);
storage::DataLayout layout;
storage::io::BufferReader reader(reinterpret_cast<const char*>(mem->Ptr()), mem->Size());
storage::io::BufferReader reader(reinterpret_cast<const char *>(mem->Ptr()), mem->Size());
storage::serialization::read(reader, layout);
std::vector<std::string> metric_prefixes;
layout.List("/mld/metrics/", std::back_inserter(metric_prefixes));
bool has_cells = false;
for (const auto &metric_prefix : metric_prefixes)
{
has_cells |= layout.HasBlock(metric_prefix + "/exclude/0/durations") &&
layout.HasBlock(metric_prefix + "/exclude/0/weights");
}
// checks that all the needed memory blocks are populated
// "/mld/cellstorage/source_boundary" and "/mld/cellstorage/destination_boundary"
// are not checked, because in situations where there are so few nodes in the graph that
// they all fit into one cell, they can be empty.
bool empty_data = layout.HasBlock("/mld/multilevelpartition/level_data") &&
layout.HasBlock("/mld/multilevelpartition/partition") &&
layout.HasBlock("/mld/multilevelpartition/cell_to_children") &&
layout.HasBlock("/mld/cellstorage/cells") &&
layout.HasBlock("/mld/cellstorage/level_to_cell_offset") &&
layout.HasBlock("/mld/multilevelgraph/node_array") &&
layout.HasBlock("/mld/multilevelgraph/edge_array") &&
layout.HasBlock("/mld/metrics/0/weights") &&
layout.HasBlock("/mld/metrics/0/durations") &&
layout.HasBlock("/mld/multilevelgraph/node_to_edge_offset");
return empty_data;
bool has_data = has_cells && layout.HasBlock("/mld/multilevelpartition/level_data") &&
layout.HasBlock("/mld/multilevelpartition/partition") &&
layout.HasBlock("/mld/multilevelpartition/cell_to_children") &&
layout.HasBlock("/mld/cellstorage/cells") &&
layout.HasBlock("/mld/cellstorage/level_to_cell_offset") &&
layout.HasBlock("/mld/multilevelgraph/node_array") &&
layout.HasBlock("/mld/multilevelgraph/edge_array") &&
layout.HasBlock("/mld/multilevelgraph/node_to_edge_offset");
return has_data;
}
else
{