Refactor cell weights and durations to own file and allow for multiple metrics

This commit is contained in:
Patrick Niklaus 2017-07-25 00:05:15 +00:00 committed by Patrick Niklaus
parent 21686ee8a9
commit 303a8fae32
24 changed files with 356 additions and 165 deletions

View File

@ -692,10 +692,10 @@ set(UTIL_LIBRARIES
# Libraries
target_link_libraries(osrm ${ENGINE_LIBRARIES})
target_link_libraries(osrm_update ${UPDATER_LIBRARIES})
target_link_libraries(osrm_contract ${CONTRACTOR_LIBRARIES} osrm_update)
target_link_libraries(osrm_contract ${CONTRACTOR_LIBRARIES} osrm_update osrm_store)
target_link_libraries(osrm_extract ${EXTRACTOR_LIBRARIES})
target_link_libraries(osrm_partition ${PARTITIONER_LIBRARIES})
target_link_libraries(osrm_customize ${CUSTOMIZER_LIBRARIES} osrm_update)
target_link_libraries(osrm_customize ${CUSTOMIZER_LIBRARIES} osrm_update osrm_store)
target_link_libraries(osrm_store ${STORAGE_LIBRARIES})
# BUILD_COMPONENTS

View File

@ -18,5 +18,5 @@ Feature: osrm-contract command line options: files
Scenario: osrm-contract - Missing input file
When I try to run "osrm-contract over-the-rainbow.osrm"
And stderr should contain "over-the-rainbow.osrm"
And stderr should contain "not found"
And stderr should contain "Missing/Broken"
And it should exit with an error

View File

@ -19,5 +19,5 @@ Feature: osrm-contract command line options: files
Scenario: osrm-customize - Missing input file
When I try to run "osrm-customize over-the-rainbow.osrm"
And stderr should contain "over-the-rainbow.osrm"
And stderr should contain "not found"
And stderr should contain "Missing/Broken"
And it should exit with an error

View File

@ -45,7 +45,7 @@ struct ContractorConfig final : storage::IOConfig
ContractorConfig()
: IOConfig(
{
".osrm",
".osrm.ebg",
},
{},
{".osrm.level", ".osrm.core", ".osrm.hsgr", ".osrm.enw"}),

View File

@ -32,9 +32,9 @@ class CellCustomizer
template <typename GraphT>
void Customize(
const GraphT &graph, Heap &heap, partition::CellStorage &cells, LevelID level, CellID id)
const GraphT &graph, Heap &heap, const partition::CellStorage &cells, CellMetric &metric, LevelID level, CellID id)
{
auto cell = cells.GetCell(level, id);
auto cell = cells.GetCell(metric, level, id);
auto destinations = cell.GetDestinationNodes();
// for each source do forward search
@ -52,9 +52,9 @@ class CellCustomizer
const EdgeDuration duration = heap.GetData(node).duration;
if (level == 1)
RelaxNode<true>(graph, cells, heap, level, node, weight, duration);
RelaxNode<true>(graph, cells, metric, heap, level, node, weight, duration);
else
RelaxNode<false>(graph, cells, heap, level, node, weight, duration);
RelaxNode<false>(graph, cells, metric, heap, level, node, weight, duration);
destinations_set.erase(node);
}
@ -80,7 +80,7 @@ class CellCustomizer
}
}
template <typename GraphT> void Customize(const GraphT &graph, partition::CellStorage &cells)
template <typename GraphT> void Customize(const GraphT &graph, const partition::CellStorage &cells, CellMetric &metric)
{
Heap heap_exemplar(graph.GetNumberOfNodes());
HeapPtr heaps(heap_exemplar);
@ -92,7 +92,7 @@ class CellCustomizer
auto &heap = heaps.local();
for (auto id = range.begin(), end = range.end(); id != end; ++id)
{
Customize(graph, heap, cells, level, id);
Customize(graph, heap, cells, metric, level, id);
}
});
}
@ -102,6 +102,7 @@ class CellCustomizer
template <bool first_level, typename GraphT>
void RelaxNode(const GraphT &graph,
const partition::CellStorage &cells,
const CellMetric &metric,
Heap &heap,
LevelID level,
NodeID node,
@ -123,7 +124,7 @@ class CellCustomizer
{
// Relax sub-cell nodes
auto subcell_id = partition.GetCell(level - 1, node);
auto subcell = cells.GetCell(level - 1, subcell_id);
auto subcell = cells.GetCell(metric, level - 1, subcell_id);
auto subcell_destination = subcell.GetDestinationNodes().begin();
auto subcell_duration = subcell.GetOutDuration(node).begin();
for (auto subcell_weight : subcell.GetOutWeight(node))

View File

@ -0,0 +1,32 @@
#ifndef OSRM_CUSTOMIZER_CELL_METRIC_HPP
#define OSRM_CUSTOMIZER_CELL_METRIC_HPP
#include "storage/io_fwd.hpp"
#include "storage/shared_memory_ownership.hpp"
#include "util/vector_view.hpp"
#include "util/typedefs.hpp"
namespace osrm
{
namespace customizer
{
namespace detail
{
// Encapsulated one metric to make it easily replacable in CelLStorage
template <storage::Ownership Ownership> struct CellMetricImpl
{
template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
Vector<EdgeWeight> weights;
Vector<EdgeDuration> durations;
};
}
using CellMetric = detail::CellMetricImpl<storage::Ownership::Container>;
using CellMetricView = detail::CellMetricImpl<storage::Ownership::View>;
}
}
#endif

View File

@ -17,12 +17,9 @@ namespace customizer
struct CustomizationConfig final : storage::IOConfig
{
CustomizationConfig()
: IOConfig(
{
".osrm",
},
{},
{".osrm.ebg", ".osrm.partition", ".osrm.cells", ".osrm.mldgr"}),
: IOConfig({".osrm.ebg", ".osrm.partition", ".osrm.cells"},
{},
{".osrm.cell_metrics", ".osrm.mldgr"}),
requested_num_threads(0)
{
}

View File

@ -0,0 +1,59 @@
#ifndef OSRM_CUSTOMIZER_FILES_HPP
#define OSRM_CUSTOMIZER_FILES_HPP
#include "customizer/serialization.hpp"
#include "storage/io.hpp"
#include "util/integer_range.hpp"
namespace osrm
{
namespace customizer
{
namespace files
{
// reads .osrm.cell_metrics file
template <typename CellMetricT>
inline void readCellMetrics(const boost::filesystem::path &path, std::vector<CellMetricT> &metrics)
{
static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
std::is_same<CellMetric, CellMetricT>::value,
"");
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
auto num_metrics = reader.ReadElementCount64();
metrics.resize(num_metrics);
for (auto& metric : metrics)
{
serialization::read(reader, metric);
}
}
// writes .osrm.cell_metrics file
template <typename CellMetricT>
inline void writeCellMetrics(const boost::filesystem::path &path, const std::vector<CellMetricT> &metrics)
{
static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
std::is_same<CellMetric, CellMetricT>::value,
"");
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.WriteElementCount64(metrics.size());
for (const auto& metric : metrics)
{
serialization::write(writer, metric);
}
}
}
}
}
#endif

View File

@ -0,0 +1,34 @@
#ifndef OSRM_CUSTOMIZER_SERIALIZATION_HPP
#define OSRM_CUSTOMIZER_SERIALIZATION_HPP
#include "partition/cell_storage.hpp"
#include "storage/io.hpp"
#include "storage/serialization.hpp"
#include "storage/shared_memory_ownership.hpp"
namespace osrm
{
namespace customizer
{
namespace serialization
{
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellMetricImpl<Ownership> &metric)
{
storage::serialization::read(reader, metric.weights);
storage::serialization::read(reader, metric.durations);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, const detail::CellMetricImpl<Ownership> &metric)
{
storage::serialization::write(writer, metric.weights);
storage::serialization::write(writer, metric.durations);
}
}
}
}
#endif

View File

@ -95,6 +95,8 @@ template <> class AlgorithmDataFacade<MLD>
virtual const partition::CellStorageView &GetCellStorage() const = 0;
virtual const customizer::CellMetricView &GetCellMetric() const = 0;
virtual EdgeRange GetBorderEdgeRange(const LevelID level, const NodeID node) const = 0;
// searches for a specific edge

View File

@ -964,6 +964,7 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
// MLD data
partition::MultiLevelPartitionView mld_partition;
partition::CellStorageView mld_cell_storage;
customizer::CellMetricView mld_cell_metric;
using QueryGraph = customizer::MultiLevelEdgeBasedGraphView;
using GraphNode = QueryGraph::NodeArrayEntry;
using GraphEdge = QueryGraph::EdgeArrayEntry;
@ -982,11 +983,6 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
char *memory_block,
const std::size_t avoid_index)
{
const auto weights_block_id = static_cast<storage::DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_WEIGHTS_0 + avoid_index);
const auto durations_block_id = static_cast<storage::DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_DURATIONS_0 + avoid_index);
if (data_layout.GetBlockSize(storage::DataLayout::MLD_PARTITION) > 0)
{
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0);
@ -1012,15 +1008,32 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
partition::MultiLevelPartitionView{level_data, partition, cell_to_children};
}
const auto weights_block_id = static_cast<storage::DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_WEIGHTS_0 + avoid_index);
const auto durations_block_id = static_cast<storage::DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_DURATIONS_0 + avoid_index);
if (data_layout.GetBlockSize(weights_block_id) > 0)
{
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0);
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0);
auto mld_cell_weights_ptr =
data_layout.GetBlockPtr<EdgeWeight>(memory_block, weights_block_id);
auto mld_cell_durations_ptr =
data_layout.GetBlockPtr<EdgeDuration>(memory_block, durations_block_id);
auto weight_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_WEIGHTS_0);
auto duration_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DURATIONS_0);
BOOST_ASSERT(weight_entries_count == duration_entries_count);
util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
util::vector_view<EdgeDuration> durations(mld_cell_durations_ptr,
duration_entries_count);
mld_cell_metric = customizer::CellMetricView {std::move(weights), std::move(durations)};
}
if (data_layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0)
{
auto mld_source_boundary_ptr = data_layout.GetBlockPtr<NodeID>(
memory_block, storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
auto mld_destination_boundary_ptr = data_layout.GetBlockPtr<NodeID>(
@ -1030,10 +1043,6 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
auto mld_cell_level_offsets_ptr = data_layout.GetBlockPtr<std::uint64_t>(
memory_block, storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
auto weight_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_WEIGHTS_0);
auto duration_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DURATIONS_0);
auto source_boundary_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
auto destination_boundary_entries_count =
@ -1042,11 +1051,6 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
auto cell_level_offsets_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
BOOST_ASSERT(weight_entries_count == duration_entries_count);
util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
util::vector_view<EdgeDuration> durations(mld_cell_durations_ptr,
duration_entries_count);
util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
source_boundary_entries_count);
util::vector_view<NodeID> destination_boundary(mld_destination_boundary_ptr,
@ -1056,9 +1060,7 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
util::vector_view<std::uint64_t> level_offsets(mld_cell_level_offsets_ptr,
cell_level_offsets_entries_count);
mld_cell_storage = partition::CellStorageView{std::move(weights),
std::move(durations),
std::move(source_boundary),
mld_cell_storage = partition::CellStorageView{std::move(source_boundary),
std::move(destination_boundary),
std::move(cells),
std::move(level_offsets)};
@ -1105,6 +1107,8 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
const partition::CellStorageView &GetCellStorage() const override { return mld_cell_storage; }
const customizer::CellMetricView &GetCellMetric() const override { return mld_cell_metric; }
// search graph access
unsigned GetNumberOfNodes() const override final { return query_graph.GetNumberOfNodes(); }

View File

@ -143,6 +143,7 @@ void routingStep(const DataFacade<Algorithm> &facade,
{
const auto &partition = facade.GetMultiLevelPartition();
const auto &cells = facade.GetCellStorage();
const auto &metric = facade.GetCellMetric();
const auto node = forward_heap.DeleteMin();
const auto weight = forward_heap.GetKey(node);
@ -174,7 +175,7 @@ void routingStep(const DataFacade<Algorithm> &facade,
if (DIRECTION == FORWARD_DIRECTION)
{
// Shortcuts in forward direction
const auto &cell = cells.GetCell(level, partition.GetCell(level, node));
const auto &cell = cells.GetCell(metric, level, partition.GetCell(level, node));
auto destination = cell.GetDestinationNodes().begin();
for (auto shortcut_weight : cell.GetOutWeight(node))
{
@ -200,7 +201,7 @@ void routingStep(const DataFacade<Algorithm> &facade,
else
{
// Shortcuts in backward direction
const auto &cell = cells.GetCell(level, partition.GetCell(level, node));
const auto &cell = cells.GetCell(metric, level, partition.GetCell(level, node));
auto source = cell.GetSourceNodes().begin();
for (auto shortcut_weight : cell.GetInWeight(node))
{

View File

@ -12,6 +12,8 @@
#include "storage/io_fwd.hpp"
#include "storage/shared_memory_ownership.hpp"
#include "customizer/cell_metric.hpp"
#include <boost/iterator/iterator_facade.hpp>
#include <boost/range/iterator_range.hpp>
#include <tbb/parallel_sort.h>
@ -89,10 +91,9 @@ template <storage::Ownership Ownership> class CellStorageImpl
WeightValueT,
boost::random_access_traversal_tag>
{
typedef boost::iterator_facade<ColumnIterator,
WeightValueT,
boost::random_access_traversal_tag>
base_t;
typedef boost::
iterator_facade<ColumnIterator, WeightValueT, boost::random_access_traversal_tag>
base_t;
public:
typedef typename base_t::value_type value_type;
@ -177,8 +178,8 @@ template <storage::Ownership Ownership> class CellStorageImpl
const NodeID *const all_destinations)
: num_source_nodes{data.num_source_nodes},
num_destination_nodes{data.num_destination_nodes},
weights{all_weights + data.value_offset},
durations{all_durations + data.value_offset},
weights{all_weights + data.value_offset}, durations{all_durations +
data.value_offset},
source_boundary{all_sources + data.source_boundary_offset},
destination_boundary{all_destinations + data.destination_boundary_offset}
{
@ -323,26 +324,42 @@ template <storage::Ownership Ownership> class CellStorageImpl
cell.value_offset = value_offset;
value_offset += cell.num_source_nodes * cell.num_destination_nodes;
}
}
weights.resize(value_offset + 1, INVALID_EDGE_WEIGHT);
durations.resize(value_offset + 1, MAXIMAL_EDGE_DURATION);
// Returns a new metric that can be used with this container
customizer::CellMetric MakeMetric() const
{
customizer::CellMetric metric;
if (cells.empty())
{
return metric;
}
const auto &last_cell = cells.back();
ValueOffset total_size = cells.back().value_offset +
last_cell.num_source_nodes * last_cell.num_destination_nodes;
metric.weights.resize(total_size + 1, INVALID_EDGE_WEIGHT);
metric.durations.resize(total_size + 1, MAXIMAL_EDGE_DURATION);
return metric;
}
template <typename = std::enable_if<Ownership == storage::Ownership::View>>
CellStorageImpl(Vector<EdgeWeight> weights_,
Vector<EdgeDuration> durations_,
Vector<NodeID> source_boundary_,
CellStorageImpl(Vector<NodeID> source_boundary_,
Vector<NodeID> destination_boundary_,
Vector<CellData> cells_,
Vector<std::uint64_t> level_to_cell_offset_)
: weights(std::move(weights_)), durations(std::move(durations_)),
source_boundary(std::move(source_boundary_)),
: source_boundary(std::move(source_boundary_)),
destination_boundary(std::move(destination_boundary_)), cells(std::move(cells_)),
level_to_cell_offset(std::move(level_to_cell_offset_))
{
}
ConstCell GetCell(LevelID level, CellID id) const
ConstCell GetCell(const customizer::detail::CellMetricImpl<Ownership> &metric,
LevelID level,
CellID id) const
{
const auto level_index = LevelIDToIndex(level);
BOOST_ASSERT(level_index < level_to_cell_offset.size());
@ -350,14 +367,14 @@ template <storage::Ownership Ownership> class CellStorageImpl
const auto cell_index = offset + id;
BOOST_ASSERT(cell_index < cells.size());
return ConstCell{cells[cell_index],
weights.data(),
durations.data(),
metric.weights.data(),
metric.durations.data(),
source_boundary.empty() ? nullptr : source_boundary.data(),
destination_boundary.empty() ? nullptr : destination_boundary.data()};
}
template <typename = std::enable_if<Ownership == storage::Ownership::Container>>
Cell GetCell(LevelID level, CellID id)
Cell GetCell(customizer::CellMetric &metric, LevelID level, CellID id) const
{
const auto level_index = LevelIDToIndex(level);
BOOST_ASSERT(level_index < level_to_cell_offset.size());
@ -365,8 +382,8 @@ template <storage::Ownership Ownership> class CellStorageImpl
const auto cell_index = offset + id;
BOOST_ASSERT(cell_index < cells.size());
return Cell{cells[cell_index],
weights.data(),
durations.data(),
metric.weights.data(),
metric.durations.data(),
source_boundary.data(),
destination_boundary.data()};
}
@ -377,8 +394,6 @@ template <storage::Ownership Ownership> class CellStorageImpl
const detail::CellStorageImpl<Ownership> &storage);
private:
Vector<EdgeWeight> weights;
Vector<EdgeDuration> durations;
Vector<NodeID> source_boundary;
Vector<NodeID> destination_boundary;
Vector<CellData> cells;

View File

@ -54,8 +54,6 @@ inline void write(storage::io::FileWriter &writer,
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage)
{
storage::serialization::read(reader, storage.weights);
storage::serialization::read(reader, storage.durations);
storage::serialization::read(reader, storage.source_boundary);
storage::serialization::read(reader, storage.destination_boundary);
storage::serialization::read(reader, storage.cells);
@ -66,13 +64,12 @@ template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::CellStorageImpl<Ownership> &storage)
{
storage::serialization::write(writer, storage.weights);
storage::serialization::write(writer, storage.durations);
storage::serialization::write(writer, storage.source_boundary);
storage::serialization::write(writer, storage.destination_boundary);
storage::serialization::write(writer, storage.cells);
storage::serialization::write(writer, storage.level_to_cell_offset);
}
}
}
}

View File

@ -66,6 +66,7 @@ struct StorageConfig final : IOConfig
".osrm.ebg_nodes",
".osrm.core",
".osrm.cells",
".osrm.cell_metrics",
".osrm.mldgr",
".osrm.tld",
".osrm.tls",

View File

@ -55,12 +55,11 @@ struct UpdaterConfig final : storage::IOConfig
".osrm.edges",
".osrm.geometry",
".osrm.fileIndex",
".osrm.datasource_names",
".osrm.properties",
".osrm.restrictions",
},
{},
{})
{".osrm.datasource_names"})
{
}

View File

@ -1,6 +1,7 @@
#include "customizer/customizer.hpp"
#include "customizer/cell_customizer.hpp"
#include "customizer/edge_based_graph.hpp"
#include "customizer/files.hpp"
#include "partition/cell_storage.hpp"
#include "partition/edge_based_graph_reader.hpp"
@ -22,7 +23,8 @@ namespace customizer
template <typename Graph, typename Partition, typename CellStorage>
void CellStorageStatistics(const Graph &graph,
const Partition &partition,
const CellStorage &storage)
const CellStorage &storage,
const CellMetric &metric)
{
util::Log() << "Cells statistics per level";
@ -38,7 +40,7 @@ void CellStorageStatistics(const Graph &graph,
std::size_t invalid_sources = 0, invalid_destinations = 0;
for (std::uint32_t cell_id = 0; cell_id < partition.GetNumberOfCells(level); ++cell_id)
{
const auto &cell = storage.GetCell(level, cell_id);
const auto &cell = storage.GetCell(metric, level, cell_id);
source += cell.GetSourceNodes().size();
destination += cell.GetDestinationNodes().size();
total += cell_nodes[cell_id];
@ -109,12 +111,13 @@ int Customizer::Run(const CustomizationConfig &config)
TIMER_START(cell_customize);
CellCustomizer customizer(mlp);
customizer.Customize(*edge_based_graph, storage);
auto metric = storage.MakeMetric();
customizer.Customize(*edge_based_graph, storage, metric);
TIMER_STOP(cell_customize);
util::Log() << "Cells customization took " << TIMER_SEC(cell_customize) << " seconds";
TIMER_START(writing_mld_data);
partition::files::writeCells(config.GetPath(".osrm.cells"), storage);
files::writeCellMetrics(config.GetPath(".osrm.cell_metrics"), std::vector<CellMetric>{metric});
TIMER_STOP(writing_mld_data);
util::Log() << "MLD customization writing took " << TIMER_SEC(writing_mld_data) << " seconds";
@ -123,7 +126,7 @@ int Customizer::Run(const CustomizationConfig &config)
TIMER_STOP(writing_graph);
util::Log() << "Graph writing took " << TIMER_SEC(writing_graph) << " seconds";
CellStorageStatistics(*edge_based_graph, mlp, storage);
CellStorageStatistics(*edge_based_graph, mlp, storage, metric);
return 0;
}

View File

@ -112,6 +112,7 @@ void relaxOutgoingEdges(const DataFacade<mld::Algorithm> &facade,
{
const auto &partition = facade.GetMultiLevelPartition();
const auto &cells = facade.GetCellStorage();
const auto &metric = facade.GetCellMetric();
auto highest_diffrent_level = [&partition, node](const SegmentID &phantom_node) {
if (phantom_node.enabled)
@ -125,7 +126,7 @@ void relaxOutgoingEdges(const DataFacade<mld::Algorithm> &facade,
if (level >= 1 && !node_data.from_clique_arc)
{
const auto &cell = cells.GetCell(level, partition.GetCell(level, node));
const auto &cell = cells.GetCell(metric, level, partition.GetCell(level, node));
if (DIRECTION == FORWARD_DIRECTION)
{ // Shortcuts in forward direction
auto destination = cell.GetDestinationNodes().begin();

View File

@ -10,6 +10,7 @@
#include "contractor/query_graph.hpp"
#include "customizer/edge_based_graph.hpp"
#include "customizer/files.hpp"
#include "extractor/class_data.hpp"
#include "extractor/compressed_edge_container.hpp"
@ -64,6 +65,8 @@ namespace osrm
namespace storage
{
static constexpr std::size_t NUM_METRICS = 8;
using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf;
using RTreeNode = util::StaticRTree<RTreeLeaf, storage::Ownership::View>::TreeNode;
using QueryGraph = util::StaticGraph<contractor::QueryEdge::EdgeData>;
@ -453,10 +456,6 @@ void Storage::PopulateLayout(DataLayout &layout)
{
io::FileReader reader(config.GetPath(".osrm.cells"), io::FileReader::VerifyFingerprint);
const auto weights_count = reader.ReadVectorSize<EdgeWeight>();
layout.SetBlockSize<EdgeWeight>(DataLayout::MLD_CELL_WEIGHTS_0, weights_count);
const auto durations_count = reader.ReadVectorSize<EdgeDuration>();
layout.SetBlockSize<EdgeDuration>(DataLayout::MLD_CELL_DURATIONS_0, durations_count);
const auto source_node_count = reader.ReadVectorSize<NodeID>();
layout.SetBlockSize<NodeID>(DataLayout::MLD_CELL_SOURCE_BOUNDARY, source_node_count);
const auto destination_node_count = reader.ReadVectorSize<NodeID>();
@ -468,22 +467,45 @@ void Storage::PopulateLayout(DataLayout &layout)
const auto level_offsets_count = reader.ReadVectorSize<std::uint64_t>();
layout.SetBlockSize<std::uint64_t>(DataLayout::MLD_CELL_LEVEL_OFFSETS,
level_offsets_count);
}
else
{
layout.SetBlockSize<char>(DataLayout::MLD_CELL_SOURCE_BOUNDARY, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_DESTINATION_BOUNDARY, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELLS, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_LEVEL_OFFSETS, 0);
}
// FIXME these need to be filled with actual weight values
layout.SetBlockSize<char>(DataLayout::MLD_CELL_WEIGHTS_1, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_WEIGHTS_2, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_WEIGHTS_3, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_WEIGHTS_4, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_WEIGHTS_5, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_WEIGHTS_6, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_WEIGHTS_7, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_DURATIONS_1, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_DURATIONS_2, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_DURATIONS_3, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_DURATIONS_4, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_DURATIONS_5, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_DURATIONS_6, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_DURATIONS_7, 0);
if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
{
io::FileReader reader(config.GetPath(".osrm.cell_metrics"),
io::FileReader::VerifyFingerprint);
auto num_metric = reader.ReadElementCount64();
if (num_metric > NUM_METRICS)
{
throw util::exception("Only " + std::to_string(NUM_METRICS) +
" metrics are supported at the same time.");
}
for (const auto index : util::irange<std::size_t>(0, num_metric))
{
const auto weights_count = reader.ReadVectorSize<EdgeWeight>();
layout.SetBlockSize<EdgeWeight>(
static_cast<DataLayout::BlockID>(DataLayout::MLD_CELL_WEIGHTS_0 + index),
weights_count);
const auto durations_count = reader.ReadVectorSize<EdgeDuration>();
layout.SetBlockSize<EdgeDuration>(
static_cast<DataLayout::BlockID>(DataLayout::MLD_CELL_DURATIONS_0 + index),
durations_count);
}
for (const auto index : util::irange<std::size_t>(num_metric, NUM_METRICS))
{
layout.SetBlockSize<EdgeWeight>(
static_cast<DataLayout::BlockID>(DataLayout::MLD_CELL_WEIGHTS_0 + index), 0);
layout.SetBlockSize<EdgeDuration>(
static_cast<DataLayout::BlockID>(DataLayout::MLD_CELL_DURATIONS_0 + index), 0);
}
}
else
{
@ -503,10 +525,6 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlockSize<char>(DataLayout::MLD_CELL_DURATIONS_5, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_DURATIONS_6, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_DURATIONS_7, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_SOURCE_BOUNDARY, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_DESTINATION_BOUNDARY, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELLS, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_LEVEL_OFFSETS, 0);
}
if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
@ -941,10 +959,6 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0);
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0);
auto mld_cell_weights_ptr = layout.GetBlockPtr<EdgeWeight, true>(
memory_ptr, storage::DataLayout::MLD_CELL_WEIGHTS_0);
auto mld_cell_duration_ptr = layout.GetBlockPtr<EdgeDuration, true>(
memory_ptr, storage::DataLayout::MLD_CELL_DURATIONS_0);
auto mld_source_boundary_ptr = layout.GetBlockPtr<NodeID, true>(
memory_ptr, storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
auto mld_destination_boundary_ptr = layout.GetBlockPtr<NodeID, true>(
@ -954,10 +968,6 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
auto mld_cell_level_offsets_ptr = layout.GetBlockPtr<std::uint64_t, true>(
memory_ptr, storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
auto weight_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_CELL_WEIGHTS_0);
auto duration_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DURATIONS_0);
auto source_boundary_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
auto destination_boundary_entries_count =
@ -966,9 +976,6 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
auto cell_level_offsets_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
util::vector_view<EdgeDuration> durations(mld_cell_duration_ptr,
duration_entries_count);
util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
source_boundary_entries_count);
util::vector_view<NodeID> destination_boundary(mld_destination_boundary_ptr,
@ -978,15 +985,44 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
util::vector_view<std::uint64_t> level_offsets(mld_cell_level_offsets_ptr,
cell_level_offsets_entries_count);
partition::CellStorageView storage{std::move(weights),
std::move(durations),
std::move(source_boundary),
partition::CellStorageView storage{std::move(source_boundary),
std::move(destination_boundary),
std::move(cells),
std::move(level_offsets)};
partition::files::readCells(config.GetPath(".osrm.cells"), storage);
}
if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
{
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0);
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0);
std::vector<customizer::CellMetricView> metrics;
for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
{
auto weights_block_id = static_cast<DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_WEIGHTS_0 + index);
auto durations_block_id = static_cast<DataLayout::BlockID>(
storage::DataLayout::MLD_CELL_DURATIONS_0 + index);
auto weight_entries_count = layout.GetBlockEntries(weights_block_id);
auto duration_entries_count = layout.GetBlockEntries(durations_block_id);
auto mld_cell_weights_ptr =
layout.GetBlockPtr<EdgeWeight, true>(memory_ptr, weights_block_id);
auto mld_cell_duration_ptr =
layout.GetBlockPtr<EdgeDuration, true>(memory_ptr, durations_block_id);
util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
util::vector_view<EdgeDuration> durations(mld_cell_duration_ptr,
duration_entries_count);
metrics.push_back(
customizer::CellMetricView{std::move(weights), std::move(durations)});
}
customizer::files::readCellMetrics(config.GetPath(".osrm.cell_metrics"), metrics);
}
if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
{

View File

@ -169,14 +169,12 @@ int main(int argc, char *argv[]) try
<< "! This setting may have performance side-effects.";
}
if (!boost::filesystem::is_regular_file(contractor_config.GetPath(".osrm")))
if (!contractor_config.IsValid())
{
util::Log(logERROR) << "Input file " << contractor_config.GetPath(".osrm").string()
<< " not found!";
return EXIT_FAILURE;
}
util::Log() << "Input file: " << contractor_config.GetPath(".osrm").filename().string();
util::Log() << "Input file: " << contractor_config.base_path.string() << ".osrm";
util::Log() << "Threads: " << contractor_config.requested_num_threads;
tbb::task_scheduler_init init(contractor_config.requested_num_threads);

View File

@ -153,10 +153,8 @@ int main(int argc, char *argv[]) try
return EXIT_FAILURE;
}
if (!boost::filesystem::is_regular_file(customization_config.GetPath(".osrm")))
if (!customization_config.IsValid())
{
util::Log(logERROR) << "Input file " << customization_config.GetPath(".osrm").string()
<< " not found!";
return EXIT_FAILURE;
}

View File

@ -59,11 +59,12 @@ BOOST_AUTO_TEST_CASE(two_level_test)
auto graph = makeGraph(mlp, edges);
CellStorage storage(mlp, graph);
auto metric = storage.MakeMetric();
CellCustomizer customizer(mlp);
CellCustomizer::Heap heap(graph.GetNumberOfNodes());
auto cell_1_0 = storage.GetCell(1, 0);
auto cell_1_1 = storage.GetCell(1, 1);
auto cell_1_0 = storage.GetCell(metric, 1, 0);
auto cell_1_1 = storage.GetCell(metric, 1, 1);
REQUIRE_SIZE_RANGE(cell_1_0.GetSourceNodes(), 1);
REQUIRE_SIZE_RANGE(cell_1_0.GetDestinationNodes(), 1);
@ -82,8 +83,8 @@ BOOST_AUTO_TEST_CASE(two_level_test)
REQUIRE_SIZE_RANGE(cell_1_1.GetOutWeight(2), 2);
REQUIRE_SIZE_RANGE(cell_1_1.GetInWeight(3), 2);
customizer.Customize(graph, heap, storage, 1, 0);
customizer.Customize(graph, heap, storage, 1, 1);
customizer.Customize(graph, heap, storage, metric, 1, 0);
customizer.Customize(graph, heap, storage, metric, 1, 1);
// cell 0
// check row source -> destination
@ -137,14 +138,15 @@ BOOST_AUTO_TEST_CASE(four_levels_test)
auto graph = makeGraph(mlp, edges);
CellStorage storage(mlp, graph);
auto metric = storage.MakeMetric();
auto cell_1_0 = storage.GetCell(1, 0);
auto cell_1_1 = storage.GetCell(1, 1);
auto cell_1_2 = storage.GetCell(1, 2);
auto cell_1_3 = storage.GetCell(1, 3);
auto cell_2_0 = storage.GetCell(2, 0);
auto cell_2_1 = storage.GetCell(2, 1);
auto cell_3_0 = storage.GetCell(3, 0);
auto cell_1_0 = storage.GetCell(metric, 1, 0);
auto cell_1_1 = storage.GetCell(metric, 1, 1);
auto cell_1_2 = storage.GetCell(metric, 1, 2);
auto cell_1_3 = storage.GetCell(metric, 1, 3);
auto cell_2_0 = storage.GetCell(metric, 2, 0);
auto cell_2_1 = storage.GetCell(metric, 2, 1);
auto cell_3_0 = storage.GetCell(metric, 3, 0);
REQUIRE_SIZE_RANGE(cell_1_0.GetSourceNodes(), 1);
REQUIRE_SIZE_RANGE(cell_1_0.GetDestinationNodes(), 1);
@ -207,13 +209,13 @@ BOOST_AUTO_TEST_CASE(four_levels_test)
CellCustomizer customizer(mlp);
CellCustomizer::Heap heap(graph.GetNumberOfNodes());
customizer.Customize(graph, heap, storage, 1, 0);
customizer.Customize(graph, heap, storage, 1, 1);
customizer.Customize(graph, heap, storage, 1, 2);
customizer.Customize(graph, heap, storage, 1, 3);
customizer.Customize(graph, heap, storage, metric, 1, 0);
customizer.Customize(graph, heap, storage, metric, 1, 1);
customizer.Customize(graph, heap, storage, metric, 1, 2);
customizer.Customize(graph, heap, storage, metric, 1, 3);
customizer.Customize(graph, heap, storage, 2, 0);
customizer.Customize(graph, heap, storage, 2, 1);
customizer.Customize(graph, heap, storage, metric, 2, 0);
customizer.Customize(graph, heap, storage, metric, 2, 1);
// level 1
// cell 0
@ -261,13 +263,19 @@ BOOST_AUTO_TEST_CASE(four_levels_test)
CHECK_EQUAL_RANGE(cell_2_1.GetInDuration(12), INVALID_EDGE_WEIGHT, 20);
CellStorage storage_rec(mlp, graph);
customizer.Customize(graph, storage_rec);
auto metric_rec = storage_rec.MakeMetric();
customizer.Customize(graph, storage_rec, metric_rec);
CHECK_EQUAL_COLLECTIONS(cell_2_1.GetOutWeight(9), storage_rec.GetCell(2, 1).GetOutWeight(9));
CHECK_EQUAL_COLLECTIONS(cell_2_1.GetOutWeight(13), storage_rec.GetCell(2, 1).GetOutWeight(13));
CHECK_EQUAL_COLLECTIONS(cell_2_1.GetInWeight(8), storage_rec.GetCell(2, 1).GetInWeight(8));
CHECK_EQUAL_COLLECTIONS(cell_2_1.GetInWeight(9), storage_rec.GetCell(2, 1).GetInWeight(9));
CHECK_EQUAL_COLLECTIONS(cell_2_1.GetInWeight(12), storage_rec.GetCell(2, 1).GetInWeight(12));
CHECK_EQUAL_COLLECTIONS(cell_2_1.GetOutWeight(9),
storage_rec.GetCell(metric_rec, 2, 1).GetOutWeight(9));
CHECK_EQUAL_COLLECTIONS(cell_2_1.GetOutWeight(13),
storage_rec.GetCell(metric_rec, 2, 1).GetOutWeight(13));
CHECK_EQUAL_COLLECTIONS(cell_2_1.GetInWeight(8),
storage_rec.GetCell(metric_rec, 2, 1).GetInWeight(8));
CHECK_EQUAL_COLLECTIONS(cell_2_1.GetInWeight(9),
storage_rec.GetCell(metric_rec, 2, 1).GetInWeight(9));
CHECK_EQUAL_COLLECTIONS(cell_2_1.GetInWeight(12),
storage_rec.GetCell(metric_rec, 2, 1).GetInWeight(12));
}
BOOST_AUTO_TEST_SUITE_END()

View File

@ -28,6 +28,8 @@ class MockBaseDataFacade : public engine::datafacade::BaseDataFacade
using StringView = util::StringView;
public:
bool AvoidNode(const NodeID ) const override { return false; };
util::Coordinate GetCoordinateOfNode(const NodeID /* id */) const override
{
return {util::FixedLongitude{0}, util::FixedLatitude{0}};

View File

@ -76,14 +76,15 @@ BOOST_AUTO_TEST_CASE(mutable_cell_storage)
// test non-const storage
CellStorage storage(mlp, graph);
auto metric = storage.MakeMetric();
// Level 1
auto cell_1_0 = storage.GetCell(1, 0);
auto cell_1_1 = storage.GetCell(1, 1);
auto cell_1_2 = storage.GetCell(1, 2);
auto cell_1_3 = storage.GetCell(1, 3);
auto cell_1_4 = storage.GetCell(1, 4);
auto cell_1_5 = storage.GetCell(1, 5);
auto cell_1_0 = storage.GetCell(metric, 1, 0);
auto cell_1_1 = storage.GetCell(metric, 1, 1);
auto cell_1_2 = storage.GetCell(metric, 1, 2);
auto cell_1_3 = storage.GetCell(metric, 1, 3);
auto cell_1_4 = storage.GetCell(metric, 1, 4);
auto cell_1_5 = storage.GetCell(metric, 1, 5);
(void)cell_1_4; // does not have border nodes
@ -108,10 +109,10 @@ BOOST_AUTO_TEST_CASE(mutable_cell_storage)
CHECK_EQUAL_RANGE(in_range_1_5_11, 3);
// Level 2
auto cell_2_0 = storage.GetCell(2, 0);
auto cell_2_1 = storage.GetCell(2, 1);
auto cell_2_2 = storage.GetCell(2, 2);
auto cell_2_3 = storage.GetCell(2, 3);
auto cell_2_0 = storage.GetCell(metric, 2, 0);
auto cell_2_1 = storage.GetCell(metric, 2, 1);
auto cell_2_2 = storage.GetCell(metric, 2, 2);
auto cell_2_3 = storage.GetCell(metric, 2, 3);
(void)cell_2_2; // does not have border nodes
@ -134,8 +135,8 @@ BOOST_AUTO_TEST_CASE(mutable_cell_storage)
CHECK_EQUAL_RANGE(in_range_2_3_11, 4);
// Level 3
auto cell_3_0 = storage.GetCell(3, 0);
auto cell_3_1 = storage.GetCell(3, 1);
auto cell_3_0 = storage.GetCell(metric, 3, 0);
auto cell_3_1 = storage.GetCell(metric, 3, 1);
auto out_range_3_0_0 = cell_3_0.GetOutWeight(0);
auto out_range_3_1_4 = cell_3_1.GetOutWeight(4);
@ -219,13 +220,14 @@ BOOST_AUTO_TEST_CASE(immutable_cell_storage)
// test const storage
const CellStorage const_storage(mlp, graph);
const auto metric =const_storage.MakeMetric();
auto const_cell_1_0 = const_storage.GetCell(1, 0);
auto const_cell_1_1 = const_storage.GetCell(1, 1);
auto const_cell_1_2 = const_storage.GetCell(1, 2);
auto const_cell_1_3 = const_storage.GetCell(1, 3);
auto const_cell_1_4 = const_storage.GetCell(1, 4);
auto const_cell_1_5 = const_storage.GetCell(1, 5);
auto const_cell_1_0 = const_storage.GetCell(metric, 1, 0);
auto const_cell_1_1 = const_storage.GetCell(metric, 1, 1);
auto const_cell_1_2 = const_storage.GetCell(metric, 1, 2);
auto const_cell_1_3 = const_storage.GetCell(metric, 1, 3);
auto const_cell_1_4 = const_storage.GetCell(metric, 1, 4);
auto const_cell_1_5 = const_storage.GetCell(metric, 1, 5);
CHECK_EQUAL_RANGE(const_cell_1_0.GetSourceNodes(), 0);
CHECK_EQUAL_COLLECTIONS(const_cell_1_1.GetSourceNodes(), std::vector<EdgeWeight>{});
@ -262,10 +264,10 @@ BOOST_AUTO_TEST_CASE(immutable_cell_storage)
REQUIRE_SIZE_RANGE(in_const_range_1_5_11, 1);
// Level 2
auto const_cell_2_0 = const_storage.GetCell(2, 0);
auto const_cell_2_1 = const_storage.GetCell(2, 1);
auto const_cell_2_2 = const_storage.GetCell(2, 2);
auto const_cell_2_3 = const_storage.GetCell(2, 3);
auto const_cell_2_0 = const_storage.GetCell(metric, 2, 0);
auto const_cell_2_1 = const_storage.GetCell(metric, 2, 1);
auto const_cell_2_2 = const_storage.GetCell(metric, 2, 2);
auto const_cell_2_3 = const_storage.GetCell(metric, 2, 3);
CHECK_EQUAL_RANGE(const_cell_2_0.GetSourceNodes(), 0);
CHECK_EQUAL_RANGE(const_cell_2_1.GetSourceNodes(), 4);
@ -296,8 +298,8 @@ BOOST_AUTO_TEST_CASE(immutable_cell_storage)
REQUIRE_SIZE_RANGE(in_const_range_2_3_7, 1);
// Level 3
auto const_cell_3_0 = const_storage.GetCell(3, 0);
auto const_cell_3_1 = const_storage.GetCell(3, 1);
auto const_cell_3_0 = const_storage.GetCell(metric, 3, 0);
auto const_cell_3_1 = const_storage.GetCell(metric, 3, 1);
CHECK_EQUAL_RANGE(const_cell_3_0.GetSourceNodes(), 0);
CHECK_EQUAL_RANGE(const_cell_3_1.GetSourceNodes(), 4, 7);
@ -321,7 +323,8 @@ BOOST_AUTO_TEST_CASE(immutable_cell_storage)
REQUIRE_SIZE_RANGE(in_const_range_3_1_4, 2);
REQUIRE_SIZE_RANGE(in_const_range_3_1_7, 2);
auto const_cell_4_0 = const_storage.GetCell(4, 0);
// Level 4
auto const_cell_4_0 = const_storage.GetCell(metric, 4, 0);
CHECK_EQUAL_COLLECTIONS(const_cell_4_0.GetSourceNodes(), std::vector<EdgeWeight>{});
CHECK_EQUAL_COLLECTIONS(const_cell_4_0.GetDestinationNodes(), std::vector<EdgeWeight>{});
}