Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Refactor cell weights and durations to own file and allow for multiple metrics

Browse Source
This commit is contained in:
Patrick Niklaus
2017-07-25 00:05:15 +00:00
committed by Patrick Niklaus
parent 21686ee8a9
commit 303a8fae32
24 changed files with 356 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/contractor/contractor_config.hpp
+1 -1
View File
@@ -45,7 +45,7 @@ struct ContractorConfig final : storage::IOConfig
ContractorConfig()
: IOConfig(
{
".osrm",
".osrm.ebg",
},
{},
{".osrm.level", ".osrm.core", ".osrm.hsgr", ".osrm.enw"}),
include/customizer/cell_customizer.hpp
+8 -7
View File
@@ -32,9 +32,9 @@ class CellCustomizer
template <typename GraphT>
void Customize(
const GraphT &graph, Heap &heap, partition::CellStorage &cells, LevelID level, CellID id)
const GraphT &graph, Heap &heap, const partition::CellStorage &cells, CellMetric &metric, LevelID level, CellID id)
{
auto cell = cells.GetCell(level, id);
auto cell = cells.GetCell(metric, level, id);
auto destinations = cell.GetDestinationNodes();
// for each source do forward search
@@ -52,9 +52,9 @@ class CellCustomizer
const EdgeDuration duration = heap.GetData(node).duration;
if (level == 1)
RelaxNode<true>(graph, cells, heap, level, node, weight, duration);
RelaxNode<true>(graph, cells, metric, heap, level, node, weight, duration);
else
RelaxNode<false>(graph, cells, heap, level, node, weight, duration);
RelaxNode<false>(graph, cells, metric, heap, level, node, weight, duration);
destinations_set.erase(node);
}
@@ -80,7 +80,7 @@ class CellCustomizer
}
}
template <typename GraphT> void Customize(const GraphT &graph, partition::CellStorage &cells)
template <typename GraphT> void Customize(const GraphT &graph, const partition::CellStorage &cells, CellMetric &metric)
{
Heap heap_exemplar(graph.GetNumberOfNodes());
HeapPtr heaps(heap_exemplar);
@@ -92,7 +92,7 @@ class CellCustomizer
auto &heap = heaps.local();
for (auto id = range.begin(), end = range.end(); id != end; ++id)
{
Customize(graph, heap, cells, level, id);
Customize(graph, heap, cells, metric, level, id);
}
});
}
@@ -102,6 +102,7 @@ class CellCustomizer
template <bool first_level, typename GraphT>
void RelaxNode(const GraphT &graph,
const partition::CellStorage &cells,
const CellMetric &metric,
Heap &heap,
LevelID level,
NodeID node,
@@ -123,7 +124,7 @@ class CellCustomizer
{
// Relax sub-cell nodes
auto subcell_id = partition.GetCell(level - 1, node);
auto subcell = cells.GetCell(level - 1, subcell_id);
auto subcell = cells.GetCell(metric, level - 1, subcell_id);
auto subcell_destination = subcell.GetDestinationNodes().begin();
auto subcell_duration = subcell.GetOutDuration(node).begin();
for (auto subcell_weight : subcell.GetOutWeight(node))
include/customizer/cell_metric.hpp
+32
View File
@@ -0,0 +1,32 @@
#ifndef OSRM_CUSTOMIZER_CELL_METRIC_HPP
#define OSRM_CUSTOMIZER_CELL_METRIC_HPP
#include "storage/io_fwd.hpp"
#include "storage/shared_memory_ownership.hpp"
#include "util/vector_view.hpp"
#include "util/typedefs.hpp"
namespace osrm
{
namespace customizer
{
namespace detail
{
// Encapsulated one metric to make it easily replacable in CelLStorage
template <storage::Ownership Ownership> struct CellMetricImpl
{
template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
Vector<EdgeWeight> weights;
Vector<EdgeDuration> durations;
};
}
using CellMetric = detail::CellMetricImpl<storage::Ownership::Container>;
using CellMetricView = detail::CellMetricImpl<storage::Ownership::View>;
}
}
#endif
include/customizer/customizer_config.hpp
+3 -6
View File
@@ -17,12 +17,9 @@ namespace customizer
struct CustomizationConfig final : storage::IOConfig
{
CustomizationConfig()
: IOConfig(
{
".osrm",
},
{},
{".osrm.ebg", ".osrm.partition", ".osrm.cells", ".osrm.mldgr"}),
: IOConfig({".osrm.ebg", ".osrm.partition", ".osrm.cells"},
{},
{".osrm.cell_metrics", ".osrm.mldgr"}),
requested_num_threads(0)
{
}
include/customizer/files.hpp
+59
View File
@@ -0,0 +1,59 @@
#ifndef OSRM_CUSTOMIZER_FILES_HPP
#define OSRM_CUSTOMIZER_FILES_HPP
#include "customizer/serialization.hpp"
#include "storage/io.hpp"
#include "util/integer_range.hpp"
namespace osrm
{
namespace customizer
{
namespace files
{
// reads .osrm.cell_metrics file
template <typename CellMetricT>
inline void readCellMetrics(const boost::filesystem::path &path, std::vector<CellMetricT> &metrics)
{
static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
std::is_same<CellMetric, CellMetricT>::value,
"");
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
auto num_metrics = reader.ReadElementCount64();
metrics.resize(num_metrics);
for (auto& metric : metrics)
{
serialization::read(reader, metric);
}
}
// writes .osrm.cell_metrics file
template <typename CellMetricT>
inline void writeCellMetrics(const boost::filesystem::path &path, const std::vector<CellMetricT> &metrics)
{
static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
std::is_same<CellMetric, CellMetricT>::value,
"");
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.WriteElementCount64(metrics.size());
for (const auto& metric : metrics)
{
serialization::write(writer, metric);
}
}
}
}
}
#endif
include/customizer/serialization.hpp
+34
View File
@@ -0,0 +1,34 @@
#ifndef OSRM_CUSTOMIZER_SERIALIZATION_HPP
#define OSRM_CUSTOMIZER_SERIALIZATION_HPP
#include "partition/cell_storage.hpp"
#include "storage/io.hpp"
#include "storage/serialization.hpp"
#include "storage/shared_memory_ownership.hpp"
namespace osrm
{
namespace customizer
{
namespace serialization
{
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellMetricImpl<Ownership> &metric)
{
storage::serialization::read(reader, metric.weights);
storage::serialization::read(reader, metric.durations);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, const detail::CellMetricImpl<Ownership> &metric)
{
storage::serialization::write(writer, metric.weights);
storage::serialization::write(writer, metric.durations);
}
}
}
}
#endif
include/engine/datafacade/algorithm_datafacade.hpp
+2
View File
@@ -95,6 +95,8 @@ template <> class AlgorithmDataFacade<MLD>
virtual const partition::CellStorageView &GetCellStorage() const = 0;
virtual const customizer::CellMetricView &GetCellMetric() const = 0;
virtual EdgeRange GetBorderEdgeRange(const LevelID level, const NodeID node) const = 0;
// searches for a specific edge
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+24 -20
View File
@@ -964,6 +964,7 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
// MLD data
partition::MultiLevelPartitionView mld_partition;
partition::CellStorageView mld_cell_storage;
customizer::CellMetricView mld_cell_metric;
using QueryGraph = customizer::MultiLevelEdgeBasedGraphView;
using GraphNode = QueryGraph::NodeArrayEntry;
using GraphEdge = QueryGraph::EdgeArrayEntry;
@@ -982,11 +983,6 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
char *memory_block,
const std::size_t avoid_index)
{
const auto weights_block_id = static_cast<storage::DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_WEIGHTS_0 + avoid_index);
const auto durations_block_id = static_cast<storage::DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_DURATIONS_0 + avoid_index);
if (data_layout.GetBlockSize(storage::DataLayout::MLD_PARTITION) > 0)
{
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0);
@@ -1012,15 +1008,32 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
partition::MultiLevelPartitionView{level_data, partition, cell_to_children};
}
const auto weights_block_id = static_cast<storage::DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_WEIGHTS_0 + avoid_index);
const auto durations_block_id = static_cast<storage::DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_DURATIONS_0 + avoid_index);
if (data_layout.GetBlockSize(weights_block_id) > 0)
{
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0);
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0);
auto mld_cell_weights_ptr =
data_layout.GetBlockPtr<EdgeWeight>(memory_block, weights_block_id);
auto mld_cell_durations_ptr =
data_layout.GetBlockPtr<EdgeDuration>(memory_block, durations_block_id);
auto weight_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_WEIGHTS_0);
auto duration_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DURATIONS_0);
BOOST_ASSERT(weight_entries_count == duration_entries_count);
util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
util::vector_view<EdgeDuration> durations(mld_cell_durations_ptr,
duration_entries_count);
mld_cell_metric = customizer::CellMetricView {std::move(weights), std::move(durations)};
}
if (data_layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0)
{
auto mld_source_boundary_ptr = data_layout.GetBlockPtr<NodeID>(
memory_block, storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
auto mld_destination_boundary_ptr = data_layout.GetBlockPtr<NodeID>(
@@ -1030,10 +1043,6 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
auto mld_cell_level_offsets_ptr = data_layout.GetBlockPtr<std::uint64_t>(
memory_block, storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
auto weight_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_WEIGHTS_0);
auto duration_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DURATIONS_0);
auto source_boundary_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
auto destination_boundary_entries_count =
@@ -1042,11 +1051,6 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
auto cell_level_offsets_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
BOOST_ASSERT(weight_entries_count == duration_entries_count);
util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
util::vector_view<EdgeDuration> durations(mld_cell_durations_ptr,
duration_entries_count);
util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
source_boundary_entries_count);
util::vector_view<NodeID> destination_boundary(mld_destination_boundary_ptr,
@@ -1056,9 +1060,7 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
util::vector_view<std::uint64_t> level_offsets(mld_cell_level_offsets_ptr,
cell_level_offsets_entries_count);
mld_cell_storage = partition::CellStorageView{std::move(weights),
std::move(durations),
std::move(source_boundary),
mld_cell_storage = partition::CellStorageView{std::move(source_boundary),
std::move(destination_boundary),
std::move(cells),
std::move(level_offsets)};
@@ -1105,6 +1107,8 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
const partition::CellStorageView &GetCellStorage() const override { return mld_cell_storage; }
const customizer::CellMetricView &GetCellMetric() const override { return mld_cell_metric; }
// search graph access
unsigned GetNumberOfNodes() const override final { return query_graph.GetNumberOfNodes(); }
include/engine/routing_algorithms/routing_base_mld.hpp
+3 -2
View File
@@ -143,6 +143,7 @@ void routingStep(const DataFacade<Algorithm> &facade,
{
const auto &partition = facade.GetMultiLevelPartition();
const auto &cells = facade.GetCellStorage();
const auto &metric = facade.GetCellMetric();
const auto node = forward_heap.DeleteMin();
const auto weight = forward_heap.GetKey(node);
@@ -174,7 +175,7 @@ void routingStep(const DataFacade<Algorithm> &facade,
if (DIRECTION == FORWARD_DIRECTION)
{
// Shortcuts in forward direction
const auto &cell = cells.GetCell(level, partition.GetCell(level, node));
const auto &cell = cells.GetCell(metric, level, partition.GetCell(level, node));
auto destination = cell.GetDestinationNodes().begin();
for (auto shortcut_weight : cell.GetOutWeight(node))
{
@@ -200,7 +201,7 @@ void routingStep(const DataFacade<Algorithm> &facade,
else
{
// Shortcuts in backward direction
const auto &cell = cells.GetCell(level, partition.GetCell(level, node));
const auto &cell = cells.GetCell(metric, level, partition.GetCell(level, node));
auto source = cell.GetSourceNodes().begin();
for (auto shortcut_weight : cell.GetInWeight(node))
{
include/partition/cell_storage.hpp
+36 -21
View File
@@ -12,6 +12,8 @@
#include "storage/io_fwd.hpp"
#include "storage/shared_memory_ownership.hpp"
#include "customizer/cell_metric.hpp"
#include <boost/iterator/iterator_facade.hpp>
#include <boost/range/iterator_range.hpp>
#include <tbb/parallel_sort.h>
@@ -89,10 +91,9 @@ template <storage::Ownership Ownership> class CellStorageImpl
WeightValueT,
boost::random_access_traversal_tag>
{
typedef boost::iterator_facade<ColumnIterator,
WeightValueT,
boost::random_access_traversal_tag>
base_t;
typedef boost::
iterator_facade<ColumnIterator, WeightValueT, boost::random_access_traversal_tag>
base_t;
public:
typedef typename base_t::value_type value_type;
@@ -177,8 +178,8 @@ template <storage::Ownership Ownership> class CellStorageImpl
const NodeID *const all_destinations)
: num_source_nodes{data.num_source_nodes},
num_destination_nodes{data.num_destination_nodes},
weights{all_weights + data.value_offset},
durations{all_durations + data.value_offset},
weights{all_weights + data.value_offset}, durations{all_durations +
data.value_offset},
source_boundary{all_sources + data.source_boundary_offset},
destination_boundary{all_destinations + data.destination_boundary_offset}
{
@@ -323,26 +324,42 @@ template <storage::Ownership Ownership> class CellStorageImpl
cell.value_offset = value_offset;
value_offset += cell.num_source_nodes * cell.num_destination_nodes;
}
}
weights.resize(value_offset + 1, INVALID_EDGE_WEIGHT);
durations.resize(value_offset + 1, MAXIMAL_EDGE_DURATION);
// Returns a new metric that can be used with this container
customizer::CellMetric MakeMetric() const
{
customizer::CellMetric metric;
if (cells.empty())
{
return metric;
}
const auto &last_cell = cells.back();
ValueOffset total_size = cells.back().value_offset +
last_cell.num_source_nodes * last_cell.num_destination_nodes;
metric.weights.resize(total_size + 1, INVALID_EDGE_WEIGHT);
metric.durations.resize(total_size + 1, MAXIMAL_EDGE_DURATION);
return metric;
}
template <typename = std::enable_if<Ownership == storage::Ownership::View>>
CellStorageImpl(Vector<EdgeWeight> weights_,
Vector<EdgeDuration> durations_,
Vector<NodeID> source_boundary_,
CellStorageImpl(Vector<NodeID> source_boundary_,
Vector<NodeID> destination_boundary_,
Vector<CellData> cells_,
Vector<std::uint64_t> level_to_cell_offset_)
: weights(std::move(weights_)), durations(std::move(durations_)),
source_boundary(std::move(source_boundary_)),
: source_boundary(std::move(source_boundary_)),
destination_boundary(std::move(destination_boundary_)), cells(std::move(cells_)),
level_to_cell_offset(std::move(level_to_cell_offset_))
{
}
ConstCell GetCell(LevelID level, CellID id) const
ConstCell GetCell(const customizer::detail::CellMetricImpl<Ownership> &metric,
LevelID level,
CellID id) const
{
const auto level_index = LevelIDToIndex(level);
BOOST_ASSERT(level_index < level_to_cell_offset.size());
@@ -350,14 +367,14 @@ template <storage::Ownership Ownership> class CellStorageImpl
const auto cell_index = offset + id;
BOOST_ASSERT(cell_index < cells.size());
return ConstCell{cells[cell_index],
weights.data(),
durations.data(),
metric.weights.data(),
metric.durations.data(),
source_boundary.empty() ? nullptr : source_boundary.data(),
destination_boundary.empty() ? nullptr : destination_boundary.data()};
}
template <typename = std::enable_if<Ownership == storage::Ownership::Container>>
Cell GetCell(LevelID level, CellID id)
Cell GetCell(customizer::CellMetric &metric, LevelID level, CellID id) const
{
const auto level_index = LevelIDToIndex(level);
BOOST_ASSERT(level_index < level_to_cell_offset.size());
@@ -365,8 +382,8 @@ template <storage::Ownership Ownership> class CellStorageImpl
const auto cell_index = offset + id;
BOOST_ASSERT(cell_index < cells.size());
return Cell{cells[cell_index],
weights.data(),
durations.data(),
metric.weights.data(),
metric.durations.data(),
source_boundary.data(),
destination_boundary.data()};
}
@@ -377,8 +394,6 @@ template <storage::Ownership Ownership> class CellStorageImpl
const detail::CellStorageImpl<Ownership> &storage);
private:
Vector<EdgeWeight> weights;
Vector<EdgeDuration> durations;
Vector<NodeID> source_boundary;
Vector<NodeID> destination_boundary;
Vector<CellData> cells;
include/partition/serialization.hpp
+1 -4
View File
@@ -54,8 +54,6 @@ inline void write(storage::io::FileWriter &writer,
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage)
{
storage::serialization::read(reader, storage.weights);
storage::serialization::read(reader, storage.durations);
storage::serialization::read(reader, storage.source_boundary);
storage::serialization::read(reader, storage.destination_boundary);
storage::serialization::read(reader, storage.cells);
@@ -66,13 +64,12 @@ template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::CellStorageImpl<Ownership> &storage)
{
storage::serialization::write(writer, storage.weights);
storage::serialization::write(writer, storage.durations);
storage::serialization::write(writer, storage.source_boundary);
storage::serialization::write(writer, storage.destination_boundary);
storage::serialization::write(writer, storage.cells);
storage::serialization::write(writer, storage.level_to_cell_offset);
}
}
}
}
include/storage/storage_config.hpp
+1
View File
@@ -66,6 +66,7 @@ struct StorageConfig final : IOConfig
".osrm.ebg_nodes",
".osrm.core",
".osrm.cells",
".osrm.cell_metrics",
".osrm.mldgr",
".osrm.tld",
".osrm.tls",
include/updater/updater_config.hpp
+1 -2
View File
@@ -55,12 +55,11 @@ struct UpdaterConfig final : storage::IOConfig
".osrm.edges",
".osrm.geometry",
".osrm.fileIndex",
".osrm.datasource_names",
".osrm.properties",
".osrm.restrictions",
},
{},
{})
{".osrm.datasource_names"})
{
}