Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Refactor metric storage

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-27 09:44:13 +00:00 committed by Patrick Niklaus
parent aec9b6a178
commit c334d11e95
14 changed files with 505 additions and 370 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
include/contractor/contracted_metric.hpp Normal file
View File

@ -0,0 +1,26 @@
#ifndef OSMR_CONTRACTOR_CONTRACTED_METRIC_HPP
#define OSMR_CONTRACTOR_CONTRACTED_METRIC_HPP
#include "contractor/query_graph.hpp"
namespace osrm
{
namespace contractor
{
namespace detail
{
template<storage::Ownership Ownership>
struct ContractedMetric
{
detail::QueryGraph<Ownership> graph;
std::vector<util::ViewOrVector<bool, Ownership>> edge_filter;
};
}
using ContractedMetric = detail::ContractedMetric<storage::Ownership::Container>;
using ContractedMetricView = detail::ContractedMetric<storage::Ownership::View>;
}
}
#endif

59
include/contractor/files.hpp
View File

@ -1,12 +1,9 @@
#ifndef OSRM_CONTRACTOR_FILES_HPP
#define OSRM_CONTRACTOR_FILES_HPP
#include "contractor/query_graph.hpp"
#include "contractor/serialization.hpp"
#include "util/serialization.hpp"
#include "storage/serialization.hpp"
#include "storage/tar.hpp"
#include <unordered_map>
namespace osrm
{
@ -15,67 +12,51 @@ namespace contractor
namespace files
{
// reads .osrm.hsgr file
template <typename QueryGraphT, typename EdgeFilterT>
template <typename ContractedMetricT>
inline void readGraph(const boost::filesystem::path &path,
unsigned &checksum,
QueryGraphT &graph,
std::vector<EdgeFilterT> &edge_filter,
std::unordered_map<std::string, ContractedMetricT> &metrics,
std::uint32_t &connectivity_checksum)
{
static_assert(std::is_same<QueryGraphView, QueryGraphT>::value ||
std::is_same<QueryGraph, QueryGraphT>::value,
"graph must be of type QueryGraph<>");
static_assert(std::is_same<EdgeFilterT, std::vector<bool>>::value ||
std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
"edge_filter must be a container of vector<bool> or vector_view<bool>");
static_assert(std::is_same<ContractedMetric, ContractedMetricT>::value ||
std::is_same<ContractedMetricView, ContractedMetricT>::value,
"metric must be of type ContractedMetric<>");
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
reader.ReadInto("/ch/checksum", checksum);
util::serialization::read(reader, "/ch/contracted_graph", graph);
reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum);
auto count = reader.ReadElementCount64("/ch/edge_filter");
edge_filter.resize(count);
for (const auto index : util::irange<std::size_t>(0, count))
for (auto &pair : metrics)
{
storage::serialization::read(
reader, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
serialization::read(reader, "/ch/metrics/" + pair.first, pair.second);
}
reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum);
}
// writes .osrm.hsgr file
template <typename QueryGraphT, typename EdgeFilterT>
template <typename ContractedMetricT>
inline void writeGraph(const boost::filesystem::path &path,
unsigned checksum,
const QueryGraphT &graph,
const std::vector<EdgeFilterT> &edge_filter,
const std::unordered_map<std::string, ContractedMetricT> &metrics,
const std::uint32_t connectivity_checksum)
{
static_assert(std::is_same<QueryGraphView, QueryGraphT>::value ||
std::is_same<QueryGraph, QueryGraphT>::value,
"graph must be of type QueryGraph<>");
static_assert(std::is_same<EdgeFilterT, std::vector<bool>>::value ||
std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
"edge_filter must be a container of vector<bool> or vector_view<bool>");
static_assert(std::is_same<ContractedMetric, ContractedMetricT>::value ||
std::is_same<ContractedMetricView, ContractedMetricT>::value,
"metric must be of type ContractedMetric<>");
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
writer.WriteElementCount64("/ch/checksum", 1);
writer.WriteFrom("/ch/checksum", checksum);
util::serialization::write(writer, "/ch/contracted_graph", graph);
writer.WriteElementCount64("/ch/edge_filter", edge_filter.size());
for (const auto index : util::irange<std::size_t>(0, edge_filter.size()))
{
storage::serialization::write(
writer, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
}
writer.WriteElementCount64("/ch/connectivity_checksum", 1);
writer.WriteFrom("/ch/connectivity_checksum", connectivity_checksum);
for (const auto &pair : metrics)
{
serialization::write(writer, "/ch/metrics/" + pair.first, pair.second);
}
}
}
}

53
include/contractor/serialization.hpp Normal file
View File

@ -0,0 +1,53 @@
#ifndef OSRM_CONTRACTOR_SERIALIZATION_HPP
#define OSRM_CONTRACTOR_SERIALIZATION_HPP
#include "contractor/contracted_metric.hpp"
#include "util/serialization.hpp"
#include "storage/serialization.hpp"
#include "storage/tar.hpp"
namespace osrm
{
namespace contractor
{
namespace serialization
{
template <storage::Ownership Ownership>
void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::ContractedMetric<Ownership> &metric)
{
util::serialization::write(writer, name + "/contracted_graph", metric.graph);
writer.WriteElementCount64(name + "/exclude", metric.edge_filter.size());
for (const auto index : util::irange<std::size_t>(0, metric.edge_filter.size()))
{
storage::serialization::write(writer,
name + "/exclude/" + std::to_string(index) + "/edge_filter",
metric.edge_filter[index]);
}
}
template <storage::Ownership Ownership>
void read(storage::tar::FileReader &reader,
const std::string &name,
detail::ContractedMetric<Ownership> &metric)
{
util::serialization::read(reader, name + "/contracted_graph", metric.graph);
metric.edge_filter.resize(reader.ReadElementCount64(name + "/exclude"));
for (const auto index : util::irange<std::size_t>(0, metric.edge_filter.size()))
{
storage::serialization::read(reader,
name + "/exclude/" + std::to_string(index) + "/edge_filter",
metric.edge_filter[index]);
}
}
}
}
}
#endif

42
include/customizer/files.hpp
View File

@ -7,6 +7,8 @@
#include "util/integer_range.hpp"
#include <unordered_map>
namespace osrm
{
namespace customizer
@ -16,7 +18,7 @@ namespace files
// reads .osrm.cell_metrics file
template <typename CellMetricT>
inline void readCellMetrics(const boost::filesystem::path &path, std::vector<CellMetricT> &metrics)
inline void readCellMetrics(const boost::filesystem::path &path, std::unordered_map<std::string, std::vector<CellMetricT>> &metrics)
{
static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
std::is_same<CellMetric, CellMetricT>::value,
@ -25,20 +27,27 @@ inline void readCellMetrics(const boost::filesystem::path &path, std::vector<Cel
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
auto num_metrics = reader.ReadElementCount64("/mld/metrics");
metrics.resize(num_metrics);
auto id = 0;
for (auto &metric : metrics)
for (auto& pair : metrics)
{
serialization::read(reader, "/mld/metrics/" + std::to_string(id++), metric);
const auto& metric_name = pair.first;
auto& metric_exclude_classes = pair.second;
auto prefix = "/mld/metrics/" + metric_name + "/exclude";
auto num_exclude_classes = reader.ReadElementCount64(prefix);
metric_exclude_classes.resize(num_exclude_classes);
auto id = 0;
for (auto &metric : metric_exclude_classes)
{
serialization::read(reader, prefix + "/" + std::to_string(id++), metric);
}
}
}
// writes .osrm.cell_metrics file
template <typename CellMetricT>
inline void writeCellMetrics(const boost::filesystem::path &path,
const std::vector<CellMetricT> &metrics)
const std::unordered_map<std::string, std::vector<CellMetricT>> &metrics)
{
static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
std::is_same<CellMetric, CellMetricT>::value,
@ -47,12 +56,19 @@ inline void writeCellMetrics(const boost::filesystem::path &path,
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
writer.WriteElementCount64("/mld/metrics", metrics.size());
auto id = 0;
for (const auto &metric : metrics)
for (const auto& pair : metrics)
{
serialization::write(writer, "/mld/metrics/" + std::to_string(id++), metric);
const auto& metric_name = pair.first;
const auto& metric_exclude_classes = pair.second;
auto prefix = "/mld/metrics/" + metric_name + "/exclude";
writer.WriteElementCount64(prefix, metric_exclude_classes.size());
auto id = 0;
for (auto &exclude_metric : metric_exclude_classes)
{
serialization::write(writer, prefix + "/" + std::to_string(id++), exclude_metric);
}
}
}
}

5
include/engine/algorithm.hpp
View File

@ -30,6 +30,11 @@ template <typename AlgorithmT> const char *name();
template <> inline const char *name<ch::Algorithm>() { return "CH"; }
template <> inline const char *name<mld::Algorithm>() { return "MLD"; }
// Algorithm identifier
template <typename AlgorithmT> const char *identifier();
template <> inline const char *identifier<ch::Algorithm>() { return "ch"; }
template <> inline const char *identifier<mld::Algorithm>() { return "mld"; }
template <typename AlgorithmT> struct HasAlternativePathSearch final : std::false_type
{
};

57
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View File

@ -83,23 +83,26 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
void InitializeGraphPointer(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
const std::string metric_prefix = "/ch/metrics/" + metric_name;
auto graph_nodes_ptr =
data_layout.GetBlockPtr<GraphNode>(memory_block, "/ch/contracted_graph/node_array");
data_layout.GetBlockPtr<GraphNode>(memory_block, metric_prefix + "/contracted_graph/node_array");
auto graph_edges_ptr =
data_layout.GetBlockPtr<GraphEdge>(memory_block, "/ch/contracted_graph/edge_array");
data_layout.GetBlockPtr<GraphEdge>(memory_block, metric_prefix + "/contracted_graph/edge_array");
auto filter_block_id = "/ch/edge_filter/" + std::to_string(exclude_index);
auto exclude_prefix = metric_prefix + "/exclude/" + std::to_string(exclude_index);
auto filter_block_id = exclude_prefix + "/edge_filter";
auto edge_filter_ptr =
data_layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_block, filter_block_id);
util::vector_view<GraphNode> node_list(
graph_nodes_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/node_array"));
graph_nodes_ptr, data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/node_array"));
util::vector_view<GraphEdge> edge_list(
graph_edges_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/edge_array"));
graph_edges_ptr, data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/edge_array"));
util::vector_view<bool> edge_filter(edge_filter_ptr,
data_layout.GetBlockEntries(filter_block_id));
@ -108,17 +111,18 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
public:
ContiguousInternalMemoryAlgorithmDataFacade(
std::shared_ptr<ContiguousBlockAllocator> allocator_, std::size_t exclude_index)
std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::string& metric_name, std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
}
void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
InitializeGraphPointer(data_layout, memory_block, exclude_index);
InitializeGraphPointer(data_layout, memory_block, metric_name, exclude_index);
}
// search graph access
@ -225,8 +229,11 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
void InitializeProfilePropertiesPointer(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
// TODO: For multi-metric support we need to have separate exclude classes per metric
(void) metric_name;
m_profile_properties = data_layout.GetBlockPtr<extractor::ProfileProperties>(
memory_block, "/common/properties");
@ -528,6 +535,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
InitializeChecksumPointer(data_layout, memory_block);
@ -538,7 +546,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
InitializeGeometryPointers(data_layout, memory_block);
InitializeNamePointers(data_layout, memory_block);
InitializeTurnLaneDescriptionsPointers(data_layout, memory_block);
InitializeProfilePropertiesPointer(data_layout, memory_block, exclude_index);
InitializeProfilePropertiesPointer(data_layout, memory_block, metric_name, exclude_index);
InitializeRTreePointers(data_layout, memory_block);
InitializeIntersectionClassPointers(data_layout, memory_block);
InitializeManeuverOverridePointers(data_layout, memory_block);
@ -548,10 +556,11 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
// allows switching between process_memory/shared_memory datafacade, based on the type of
// allocator
ContiguousInternalMemoryDataFacadeBase(std::shared_ptr<ContiguousBlockAllocator> allocator_,
const std::string& metric_name,
const std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
}
// node and edge information access
@ -959,9 +968,10 @@ class ContiguousInternalMemoryDataFacade<CH>
{
public:
ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator,
const std::string& metric_name,
const std::size_t exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<CH>(allocator, exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<CH>(allocator, metric_name, exclude_index)
{
}
@ -981,14 +991,16 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
InitializeMLDDataPointers(data_layout, memory_block, exclude_index);
InitializeMLDDataPointers(data_layout, memory_block, metric_name, exclude_index);
InitializeGraphPointer(data_layout, memory_block);
}
void InitializeMLDDataPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
if (data_layout.GetBlockSize("/mld/multilevelpartition/partition") > 0)
@ -1016,9 +1028,9 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
partitioner::MultiLevelPartitionView{level_data, partition, cell_to_children};
}
const auto weights_block_id = "/mld/metrics/" + std::to_string(exclude_index) + "/weights";
const auto durations_block_id =
"/mld/metrics/" + std::to_string(exclude_index) + "/durations";
const auto exclude_prefix = "/mld/metrics/" + metric_name + "/exclude/" + std::to_string(exclude_index);
const auto weights_block_id = exclude_prefix + "/weights";
const auto durations_block_id = exclude_prefix + "/durations";
if (data_layout.GetBlockSize(weights_block_id) > 0)
{
@ -1026,8 +1038,8 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
data_layout.GetBlockPtr<EdgeWeight>(memory_block, weights_block_id);
auto mld_cell_durations_ptr =
data_layout.GetBlockPtr<EdgeDuration>(memory_block, durations_block_id);
auto weight_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/weights");
auto duration_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/durations");
auto weight_entries_count = data_layout.GetBlockEntries(weights_block_id);
auto duration_entries_count = data_layout.GetBlockEntries(durations_block_id);
BOOST_ASSERT(weight_entries_count == duration_entries_count);
util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
util::vector_view<EdgeDuration> durations(mld_cell_durations_ptr,
@ -1099,10 +1111,10 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
public:
ContiguousInternalMemoryAlgorithmDataFacade(
std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::size_t exclude_index)
std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::string& metric_name, const std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
}
const partitioner::MultiLevelPartitionView &GetMultiLevelPartition() const override
@ -1156,9 +1168,10 @@ class ContiguousInternalMemoryDataFacade<MLD> final
private:
public:
ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator,
const std::string& metric_name,
const std::size_t exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<MLD>(allocator, exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<MLD>(allocator, metric_name, exclude_index)
{
}

35
include/engine/datafacade_factory.hpp
View File

@ -33,6 +33,7 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
DataFacadeFactory(std::shared_ptr<AllocatorT> allocator)
: DataFacadeFactory(allocator, has_exclude_flags)
{
BOOST_ASSERT_MSG(facades.size() >= 1, "At least one datafacade is needed");
}
template <typename ParameterT> std::shared_ptr<const Facade> Get(const ParameterT &params) const
@ -45,13 +46,27 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
template <typename AllocatorT>
DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::true_type)
{
for (const auto index : util::irange<std::size_t>(0, facades.size()))
{
facades[index] = std::make_shared<const Facade>(allocator, index);
}
properties = allocator->GetLayout().template GetBlockPtr<extractor::ProfileProperties>(
const auto &layout = allocator->GetLayout();
properties = layout.template GetBlockPtr<extractor::ProfileProperties>(
allocator->GetMemory(), "/common/properties");
const auto &metric_name = properties->GetWeightName();
std::vector<std::string> exclude_prefixes;
auto exclude_path = std::string("/") + routing_algorithms::identifier<AlgorithmT>() +
std::string("/metrics/") + metric_name + "/exclude/";
layout.List(exclude_path, std::back_inserter(exclude_prefixes));
facades.resize(exclude_prefixes.size());
for (const auto &exclude_prefix : exclude_prefixes)
{
auto index_begin = exclude_prefix.find_last_of("/");
BOOST_ASSERT_MSG(index_begin != std::string::npos,
"The exclude prefix needs to be a valid data path.");
std::size_t index =
std::stoi(exclude_prefix.substr(index_begin + 1, exclude_prefix.size()));
BOOST_ASSERT(index >= 0 && index < facades.size());
facades[index] = std::make_shared<const Facade>(allocator, metric_name, index);
}
for (const auto index : util::irange<std::size_t>(0, properties->class_names.size()))
{
@ -67,7 +82,11 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
template <typename AllocatorT>
DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::false_type)
{
facades[0] = std::make_shared<const Facade>(allocator, 0);
const auto &layout = allocator->GetLayout();
properties = layout.template GetBlockPtr<extractor::ProfileProperties>(
allocator->GetMemory(), "/common/properties");
const auto &metric_name = properties->GetWeightName();
facades.push_back(std::make_shared<const Facade>(allocator, metric_name, 0));
}
std::shared_ptr<const Facade> Get(const api::TileParameters &, std::false_type) const
@ -124,7 +143,7 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
return {};
}
std::array<std::shared_ptr<const Facade>, extractor::MAX_EXCLUDABLE_CLASSES> facades;
std::vector<std::shared_ptr<const Facade>> facades;
std::unordered_map<std::string, extractor::ClassData> name_to_class;
const extractor::ProfileProperties *properties = nullptr;
};

47
include/engine/engine.hpp
View File

@ -155,10 +155,19 @@ bool Engine<routing_algorithms::ch::Algorithm>::CheckCompatibility(const EngineC
auto mem = storage::makeSharedMemory(barrier.data().region);
storage::DataLayout layout;
storage::io::BufferReader reader(reinterpret_cast<const char*>(mem->Ptr()), mem->Size());
storage::io::BufferReader reader(reinterpret_cast<const char *>(mem->Ptr()), mem->Size());
storage::serialization::read(reader, layout);
return layout.HasBlock("/ch/contracted_graph/node_array") &&
layout.HasBlock("/ch/contracted_graph/edge_array");
std::vector<std::string> metric_prefixes;
layout.List("/ch/metrics/", std::back_inserter(metric_prefixes));
bool has_graph = false;
for (const auto &metric_prefix : metric_prefixes)
{
has_graph |= layout.HasBlock(metric_prefix + "/contracted_graph/node_array") &&
layout.HasBlock(metric_prefix + "/contracted_graph/edge_array");
}
return has_graph;
}
else
{
@ -177,24 +186,32 @@ bool Engine<routing_algorithms::mld::Algorithm>::CheckCompatibility(const Engine
auto mem = storage::makeSharedMemory(barrier.data().region);
storage::DataLayout layout;
storage::io::BufferReader reader(reinterpret_cast<const char*>(mem->Ptr()), mem->Size());
storage::io::BufferReader reader(reinterpret_cast<const char *>(mem->Ptr()), mem->Size());
storage::serialization::read(reader, layout);
std::vector<std::string> metric_prefixes;
layout.List("/mld/metrics/", std::back_inserter(metric_prefixes));
bool has_cells = false;
for (const auto &metric_prefix : metric_prefixes)
{
has_cells |= layout.HasBlock(metric_prefix + "/exclude/0/durations") &&
layout.HasBlock(metric_prefix + "/exclude/0/weights");
}
// checks that all the needed memory blocks are populated
// "/mld/cellstorage/source_boundary" and "/mld/cellstorage/destination_boundary"
// are not checked, because in situations where there are so few nodes in the graph that
// they all fit into one cell, they can be empty.
bool empty_data = layout.HasBlock("/mld/multilevelpartition/level_data") &&
layout.HasBlock("/mld/multilevelpartition/partition") &&
layout.HasBlock("/mld/multilevelpartition/cell_to_children") &&
layout.HasBlock("/mld/cellstorage/cells") &&
layout.HasBlock("/mld/cellstorage/level_to_cell_offset") &&
layout.HasBlock("/mld/multilevelgraph/node_array") &&
layout.HasBlock("/mld/multilevelgraph/edge_array") &&
layout.HasBlock("/mld/metrics/0/weights") &&
layout.HasBlock("/mld/metrics/0/durations") &&
layout.HasBlock("/mld/multilevelgraph/node_to_edge_offset");
return empty_data;
bool has_data = has_cells && layout.HasBlock("/mld/multilevelpartition/level_data") &&
layout.HasBlock("/mld/multilevelpartition/partition") &&
layout.HasBlock("/mld/multilevelpartition/cell_to_children") &&
layout.HasBlock("/mld/cellstorage/cells") &&
layout.HasBlock("/mld/cellstorage/level_to_cell_offset") &&
layout.HasBlock("/mld/multilevelgraph/node_array") &&
layout.HasBlock("/mld/multilevelgraph/edge_array") &&
layout.HasBlock("/mld/multilevelgraph/node_to_edge_offset");
return has_data;
}
else
{

131
include/storage/serialization.hpp
View File

@ -27,6 +27,72 @@ namespace storage
namespace serialization
{
namespace detail
{
template <typename T, typename BlockT = unsigned char>
inline BlockT packBits(const T &data, std::size_t index, std::size_t count)
{
static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
BlockT value = 0;
for (std::size_t bit = 0; bit < count; ++bit, ++index)
value = (value << 1) | data[index];
return value;
}
template <typename T, typename BlockT = unsigned char>
inline void unpackBits(T &data, std::size_t index, std::size_t count, BlockT value)
{
static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
const BlockT mask = BlockT{1} << (count - 1);
for (std::size_t bit = 0; bit < count; value <<= 1, ++bit, ++index)
data[index] = value & mask;
}
template <typename VectorT>
void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data)
{
const auto count = reader.ReadElementCount64(name);
data.resize(count);
std::uint64_t index = 0;
constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
const auto decode = [&](const std::uint64_t block) {
auto read_size = std::min<std::size_t>(count - index, WORD_BITS);
unpackBits<VectorT, std::uint64_t>(data, index, read_size, block);
index += WORD_BITS;
};
reader.ReadStreaming<std::uint64_t>(name, boost::make_function_output_iterator(decode));
}
template <typename VectorT>
void writeBoolVector(tar::FileWriter &writer, const std::string &name, const VectorT &data)
{
const auto count = data.size();
writer.WriteElementCount64(name, count);
std::uint64_t index = 0;
constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
// FIXME on old boost version the function_input_iterator does not work with lambdas
// so we need to wrap it in a function here.
const std::function<std::uint64_t()> encode_function = [&]() -> std::uint64_t {
auto write_size = std::min<std::size_t>(count - index, WORD_BITS);
auto packed = packBits<VectorT, std::uint64_t>(data, index, write_size);
index += WORD_BITS;
return packed;
};
std::uint64_t number_of_blocks = (count + WORD_BITS - 1) / WORD_BITS;
writer.WriteStreaming<std::uint64_t>(
name,
boost::make_function_input_iterator(encode_function, boost::infinite()),
number_of_blocks);
}
}
/* All vector formats here use the same on-disk format.
* This is important because we want to be able to write from a vector
* of one kind, but read it into a vector of another kind.
@ -155,71 +221,6 @@ void write(tar::FileWriter &writer, const std::string &name, const util::vector_
writer.WriteFrom(name, data.data(), count);
}
namespace detail
{
template <typename T, typename BlockT = unsigned char>
inline BlockT packBits(const T &data, std::size_t index, std::size_t count)
{
static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
BlockT value = 0;
for (std::size_t bit = 0; bit < count; ++bit, ++index)
value = (value << 1) | data[index];
return value;
}
template <typename T, typename BlockT = unsigned char>
inline void unpackBits(T &data, std::size_t index, std::size_t count, BlockT value)
{
static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
const BlockT mask = BlockT{1} << (count - 1);
for (std::size_t bit = 0; bit < count; value <<= 1, ++bit, ++index)
data[index] = value & mask;
}
template <typename VectorT>
void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data)
{
const auto count = reader.ReadElementCount64(name);
data.resize(count);
std::uint64_t index = 0;
constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
const auto decode = [&](const std::uint64_t block) {
auto read_size = std::min<std::size_t>(count - index, WORD_BITS);
unpackBits<VectorT, std::uint64_t>(data, index, read_size, block);
index += WORD_BITS;
};
reader.ReadStreaming<std::uint64_t>(name, boost::make_function_output_iterator(decode));
}
template <typename VectorT>
void writeBoolVector(tar::FileWriter &writer, const std::string &name, const VectorT &data)
{
const auto count = data.size();
writer.WriteElementCount64(name, count);
std::uint64_t index = 0;
constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
// FIXME on old boost version the function_input_iterator does not work with lambdas
// so we need to wrap it in a function here.
const std::function<std::uint64_t()> encode_function = [&]() -> std::uint64_t {
auto write_size = std::min<std::size_t>(count - index, WORD_BITS);
auto packed = packBits<VectorT, std::uint64_t>(data, index, write_size);
index += WORD_BITS;
return packed;
};
std::uint64_t number_of_blocks = (count + WORD_BITS - 1) / WORD_BITS;
writer.WriteStreaming<std::uint64_t>(
name,
boost::make_function_input_iterator(encode_function, boost::infinite()),
number_of_blocks);
}
}
template <>
inline void
read<bool>(tar::FileReader &reader, const std::string &name, util::vector_view<bool> &data)

2
include/storage/shared_datatype.hpp
View File

@ -94,8 +94,6 @@ class DataLayout
{
char *start_canary_ptr = ptr - sizeof(CANARY);
char *end_canary_ptr = ptr + GetBlockSize(name);
std::cout << name << ": " << (long)(start_canary_ptr - shared_memory) << " -> "
<< (long)(end_canary_ptr - shared_memory) << std::endl;
std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
}

8
src/contractor/contractor.cpp
View File

@ -79,6 +79,7 @@ int Contractor::Run()
TIMER_START(contraction);
std::string metric_name;
std::vector<std::vector<bool>> node_filters;
{
extractor::EdgeBasedNodeDataContainer node_data;
@ -86,6 +87,7 @@ int Contractor::Run()
extractor::ProfileProperties properties;
extractor::files::readProfileProperties(config.GetPath(".osrm.properties"), properties);
metric_name = properties.GetWeightName();
node_filters =
util::excludeFlagsToNodeFilter(number_of_edge_based_nodes, node_data, properties);
@ -105,8 +107,10 @@ int Contractor::Run()
util::Log() << "Contracted graph has " << query_graph.GetNumberOfEdges() << " edges.";
util::Log() << "Contraction took " << TIMER_SEC(contraction) << " sec";
files::writeGraph(
config.GetPath(".osrm.hsgr"), checksum, query_graph, edge_filters, connectivity_checksum);
std::unordered_map<std::string, ContractedMetric> metrics = {
{metric_name, {std::move(query_graph), std::move(edge_filters)}}};
files::writeGraph(config.GetPath(".osrm.hsgr"), checksum, metrics, connectivity_checksum);
TIMER_STOP(preparing);

5
src/customize/customizer.cpp
View File

@ -153,7 +153,10 @@ int Customizer::Run(const CustomizationConfig &config)
util::Log() << "Cells customization took " << TIMER_SEC(cell_customize) << " seconds";
TIMER_START(writing_mld_data);
files::writeCellMetrics(config.GetPath(".osrm.cell_metrics"), metrics);
std::unordered_map<std::string, std::vector<CellMetric>> metric_exclude_classes = {
{properties.GetWeightName(), std::move(metrics)},
};
files::writeCellMetrics(config.GetPath(".osrm.cell_metrics"), metric_exclude_classes);
TIMER_STOP(writing_mld_data);
util::Log() << "MLD customization writing took " << TIMER_SEC(writing_mld_data) << " seconds";

372
src/storage/storage.cpp
View File

@ -554,12 +554,16 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
extractor::files::readRamIndex(config.GetPath(".osrm.ramIndex"), rtree);
}
// FIXME we only need to get the weight name
std::string metric_name;
// load profile properties
{
const auto profile_properties_ptr = layout.GetBlockPtr<extractor::ProfileProperties, true>(
memory_ptr, "/common/properties");
extractor::files::readProfileProperties(config.GetPath(".osrm.properties"),
*profile_properties_ptr);
metric_name = profile_properties_ptr->GetWeightName();
}
// Load intersection data
@ -604,210 +608,198 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes);
}
{ // Load the HSGR file
if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
{
const std::string metric_prefix = "/ch/metrics/" + metric_name;
auto graph_nodes_ptr = layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, metric_prefix + "/contracted_graph/node_array");
auto graph_edges_ptr = layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, metric_prefix + "/contracted_graph/edge_array");
auto checksum = layout.GetBlockPtr<unsigned, true>(memory_ptr, "/ch/checksum");
util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
graph_nodes_ptr,
layout.GetBlockEntries(metric_prefix + "/contracted_graph/node_array"));
util::vector_view<contractor::QueryGraphView::EdgeArrayEntry> edge_list(
graph_edges_ptr,
layout.GetBlockEntries(metric_prefix + "/contracted_graph/edge_array"));
std::vector<util::vector_view<bool>> edge_filter;
layout.List(
metric_prefix + "/exclude", boost::make_function_output_iterator([&](const auto &name) {
auto data_ptr =
layout.GetBlockPtr<util::vector_view<bool>::Word, true>(memory_ptr, name);
auto num_entries = layout.GetBlockEntries(name);
edge_filter.emplace_back(data_ptr, num_entries);
}));
std::uint32_t graph_connectivity_checksum = 0;
std::unordered_map<std::string, contractor::ContractedMetricView> metrics = {
{metric_name, {{std::move(node_list), std::move(edge_list)}, std::move(edge_filter)}}};
contractor::files::readGraph(
config.GetPath(".osrm.hsgr"), *checksum, metrics, graph_connectivity_checksum);
if (turns_connectivity_checksum != graph_connectivity_checksum)
{
auto graph_nodes_ptr =
layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, "/ch/contracted_graph/node_array");
auto graph_edges_ptr =
layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, "/ch/contracted_graph/edge_array");
auto checksum = layout.GetBlockPtr<unsigned, true>(memory_ptr, "/ch/checksum");
util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
graph_nodes_ptr, layout.GetBlockEntries("/ch/contracted_graph/node_array"));
util::vector_view<contractor::QueryGraphView::EdgeArrayEntry> edge_list(
graph_edges_ptr, layout.GetBlockEntries("/ch/contracted_graph/edge_array"));
std::vector<util::vector_view<bool>> edge_filter;
layout.List(
"/ch/edge_filter", boost::make_function_output_iterator([&](const auto &name) {
auto data_ptr =
layout.GetBlockPtr<util::vector_view<bool>::Word, true>(memory_ptr, name);
auto num_entries = layout.GetBlockEntries(name);
edge_filter.emplace_back(data_ptr, num_entries);
}));
std::uint32_t graph_connectivity_checksum = 0;
contractor::QueryGraphView graph_view(std::move(node_list), std::move(edge_list));
contractor::files::readGraph(config.GetPath(".osrm.hsgr"),
*checksum,
graph_view,
edge_filter,
graph_connectivity_checksum);
if (turns_connectivity_checksum != graph_connectivity_checksum)
{
throw util::exception(
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) +
" in " + config.GetPath(".osrm.hsgr").string() +
" does not equal to checksum " + std::to_string(turns_connectivity_checksum) +
" in " + config.GetPath(".osrm.edges").string());
}
}
else
{
layout.GetBlockPtr<unsigned, true>(memory_ptr, "/ch/checksum");
layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
memory_ptr, "/ch/contracted_graph/node_array");
layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
memory_ptr, "/ch/contracted_graph/edge_array");
throw util::exception(
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) + " in " +
config.GetPath(".osrm.hsgr").string() + " does not equal to checksum " +
std::to_string(turns_connectivity_checksum) + " in " +
config.GetPath(".osrm.edges").string());
}
}
{ // Loading MLD Data
if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
{
BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/level_data") > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/partition") > 0);
auto level_data = layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData, true>(
memory_ptr, "/mld/multilevelpartition/level_data");
auto mld_partition_ptr =
layout.GetBlockPtr<PartitionID, true>(memory_ptr, "/mld/multilevelpartition/partition");
auto partition_entries_count = layout.GetBlockEntries("/mld/multilevelpartition/partition");
util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count);
auto mld_chilren_ptr = layout.GetBlockPtr<CellID, true>(
memory_ptr, "/mld/multilevelpartition/cell_to_children");
auto children_entries_count =
layout.GetBlockEntries("/mld/multilevelpartition/cell_to_children");
util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count);
partitioner::MultiLevelPartitionView mlp{
std::move(level_data), std::move(partition), std::move(cell_to_children)};
partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp);
}
if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
{
BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
auto mld_source_boundary_ptr =
layout.GetBlockPtr<NodeID, true>(memory_ptr, "/mld/cellstorage/source_boundary");
auto mld_destination_boundary_ptr =
layout.GetBlockPtr<NodeID, true>(memory_ptr, "/mld/cellstorage/destination_boundary");
auto mld_cells_ptr = layout.GetBlockPtr<partitioner::CellStorageView::CellData, true>(
memory_ptr, "/mld/cellstorage/cells");
auto mld_cell_level_offsets_ptr = layout.GetBlockPtr<std::uint64_t, true>(
memory_ptr, "/mld/cellstorage/level_to_cell_offset");
auto source_boundary_entries_count =
layout.GetBlockEntries("/mld/cellstorage/source_boundary");
auto destination_boundary_entries_count =
layout.GetBlockEntries("/mld/cellstorage/destination_boundary");
auto cells_entries_counts = layout.GetBlockEntries("/mld/cellstorage/cells");
auto cell_level_offsets_entries_count =
layout.GetBlockEntries("/mld/cellstorage/level_to_cell_offset");
util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
source_boundary_entries_count);
util::vector_view<NodeID> destination_boundary(mld_destination_boundary_ptr,
destination_boundary_entries_count);
util::vector_view<partitioner::CellStorageView::CellData> cells(mld_cells_ptr,
cells_entries_counts);
util::vector_view<std::uint64_t> level_offsets(mld_cell_level_offsets_ptr,
cell_level_offsets_entries_count);
partitioner::CellStorageView storage{std::move(source_boundary),
std::move(destination_boundary),
std::move(cells),
std::move(level_offsets)};
partitioner::files::readCells(config.GetPath(".osrm.cells"), storage);
}
if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
{
BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
std::unordered_map<std::string, std::vector<customizer::CellMetricView>> metrics = {
{metric_name, {}},
};
std::vector<std::string> metric_prefix_names;
layout.List("/mld/metrics/" + metric_name + "/exclude/",
std::back_inserter(metric_prefix_names));
for (const auto &prefix : metric_prefix_names)
{
BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/level_data") > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/partition") > 0);
auto weights_block_id = prefix + "/weights";
auto durations_block_id = prefix + "/durations";
auto level_data =
layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData, true>(
memory_ptr, "/mld/multilevelpartition/level_data");
auto weight_entries_count = layout.GetBlockEntries(weights_block_id);
auto duration_entries_count = layout.GetBlockEntries(durations_block_id);
auto mld_cell_weights_ptr =
layout.GetBlockPtr<EdgeWeight, true>(memory_ptr, weights_block_id);
auto mld_cell_duration_ptr =
layout.GetBlockPtr<EdgeDuration, true>(memory_ptr, durations_block_id);
util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
util::vector_view<EdgeDuration> durations(mld_cell_duration_ptr,
duration_entries_count);
auto mld_partition_ptr = layout.GetBlockPtr<PartitionID, true>(
memory_ptr, "/mld/multilevelpartition/partition");
auto partition_entries_count =
layout.GetBlockEntries("/mld/multilevelpartition/partition");
util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count);
auto mld_chilren_ptr = layout.GetBlockPtr<CellID, true>(
memory_ptr, "/mld/multilevelpartition/cell_to_children");
auto children_entries_count =
layout.GetBlockEntries("/mld/multilevelpartition/cell_to_children");
util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count);
partitioner::MultiLevelPartitionView mlp{
std::move(level_data), std::move(partition), std::move(cell_to_children)};
partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp);
metrics[metric_name].push_back(
customizer::CellMetricView{std::move(weights), std::move(durations)});
}
if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
customizer::files::readCellMetrics(config.GetPath(".osrm.cell_metrics"), metrics);
}
if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
{
auto graph_nodes_ptr =
layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry, true>(
memory_ptr, "/mld/multilevelgraph/node_array");
auto graph_edges_ptr =
layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry, true>(
memory_ptr, "/mld/multilevelgraph/edge_array");
auto graph_node_to_offset_ptr =
layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset, true>(
memory_ptr, "/mld/multilevelgraph/node_to_edge_offset");
util::vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry> node_list(
graph_nodes_ptr, layout.GetBlockEntries("/mld/multilevelgraph/node_array"));
util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry> edge_list(
graph_edges_ptr, layout.GetBlockEntries("/mld/multilevelgraph/edge_array"));
util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset> node_to_offset(
graph_node_to_offset_ptr,
layout.GetBlockEntries("/mld/multilevelgraph/node_to_edge_offset"));
std::uint32_t graph_connectivity_checksum = 0;
customizer::MultiLevelEdgeBasedGraphView graph_view(
std::move(node_list), std::move(edge_list), std::move(node_to_offset));
partitioner::files::readGraph(
config.GetPath(".osrm.mldgr"), graph_view, graph_connectivity_checksum);
if (turns_connectivity_checksum != graph_connectivity_checksum)
{
BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
auto mld_source_boundary_ptr =
layout.GetBlockPtr<NodeID, true>(memory_ptr, "/mld/cellstorage/source_boundary");
auto mld_destination_boundary_ptr = layout.GetBlockPtr<NodeID, true>(
memory_ptr, "/mld/cellstorage/destination_boundary");
auto mld_cells_ptr = layout.GetBlockPtr<partitioner::CellStorageView::CellData, true>(
memory_ptr, "/mld/cellstorage/cells");
auto mld_cell_level_offsets_ptr = layout.GetBlockPtr<std::uint64_t, true>(
memory_ptr, "/mld/cellstorage/level_to_cell_offset");
auto source_boundary_entries_count =
layout.GetBlockEntries("/mld/cellstorage/source_boundary");
auto destination_boundary_entries_count =
layout.GetBlockEntries("/mld/cellstorage/destination_boundary");
auto cells_entries_counts = layout.GetBlockEntries("/mld/cellstorage/cells");
auto cell_level_offsets_entries_count =
layout.GetBlockEntries("/mld/cellstorage/level_to_cell_offset");
util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
source_boundary_entries_count);
util::vector_view<NodeID> destination_boundary(mld_destination_boundary_ptr,
destination_boundary_entries_count);
util::vector_view<partitioner::CellStorageView::CellData> cells(mld_cells_ptr,
cells_entries_counts);
util::vector_view<std::uint64_t> level_offsets(mld_cell_level_offsets_ptr,
cell_level_offsets_entries_count);
partitioner::CellStorageView storage{std::move(source_boundary),
std::move(destination_boundary),
std::move(cells),
std::move(level_offsets)};
partitioner::files::readCells(config.GetPath(".osrm.cells"), storage);
throw util::exception(
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) + " in " +
config.GetPath(".osrm.mldgr").string() + " does not equal to checksum " +
std::to_string(turns_connectivity_checksum) + " in " +
config.GetPath(".osrm.edges").string());
}
}
if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
{
BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
// load maneuver overrides
{
const auto maneuver_overrides_ptr =
layout.GetBlockPtr<extractor::StorageManeuverOverride, true>(
memory_ptr, "/common/maneuver_overrides/overrides");
const auto maneuver_override_node_sequences_ptr = layout.GetBlockPtr<NodeID, true>(
memory_ptr, "/common/maneuver_overrides/node_sequences");
std::vector<customizer::CellMetricView> metrics;
util::vector_view<extractor::StorageManeuverOverride> maneuver_overrides(
maneuver_overrides_ptr, layout.GetBlockEntries("/common/maneuver_overrides/overrides"));
util::vector_view<NodeID> maneuver_override_node_sequences(
maneuver_override_node_sequences_ptr,
layout.GetBlockEntries("/common/maneuver_overrides/node_sequences"));
std::vector<std::string> metric_prefix_names;
layout.List("/mld/metrics/", std::back_inserter(metric_prefix_names));
for (const auto &prefix : metric_prefix_names)
{
auto weights_block_id = prefix + "/weights";
auto durations_block_id = prefix + "/durations";
auto weight_entries_count = layout.GetBlockEntries(weights_block_id);
auto duration_entries_count = layout.GetBlockEntries(durations_block_id);
auto mld_cell_weights_ptr =
layout.GetBlockPtr<EdgeWeight, true>(memory_ptr, weights_block_id);
auto mld_cell_duration_ptr =
layout.GetBlockPtr<EdgeDuration, true>(memory_ptr, durations_block_id);
util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
util::vector_view<EdgeDuration> durations(mld_cell_duration_ptr,
duration_entries_count);
metrics.push_back(
customizer::CellMetricView{std::move(weights), std::move(durations)});
}
customizer::files::readCellMetrics(config.GetPath(".osrm.cell_metrics"), metrics);
}
if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
{
auto graph_nodes_ptr =
layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry, true>(
memory_ptr, "/mld/multilevelgraph/node_array");
auto graph_edges_ptr =
layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry, true>(
memory_ptr, "/mld/multilevelgraph/edge_array");
auto graph_node_to_offset_ptr =
layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset, true>(
memory_ptr, "/mld/multilevelgraph/node_to_edge_offset");
util::vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry> node_list(
graph_nodes_ptr, layout.GetBlockEntries("/mld/multilevelgraph/node_array"));
util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry> edge_list(
graph_edges_ptr, layout.GetBlockEntries("/mld/multilevelgraph/edge_array"));
util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset> node_to_offset(
graph_node_to_offset_ptr,
layout.GetBlockEntries("/mld/multilevelgraph/node_to_edge_offset"));
std::uint32_t graph_connectivity_checksum = 0;
customizer::MultiLevelEdgeBasedGraphView graph_view(
std::move(node_list), std::move(edge_list), std::move(node_to_offset));
partitioner::files::readGraph(
config.GetPath(".osrm.mldgr"), graph_view, graph_connectivity_checksum);
if (turns_connectivity_checksum != graph_connectivity_checksum)
{
throw util::exception(
"Connectivity checksum " + std::to_string(graph_connectivity_checksum) +
" in " + config.GetPath(".osrm.mldgr").string() +
" does not equal to checksum " + std::to_string(turns_connectivity_checksum) +
" in " + config.GetPath(".osrm.edges").string());
}
}
// load maneuver overrides
{
const auto maneuver_overrides_ptr =
layout.GetBlockPtr<extractor::StorageManeuverOverride, true>(
memory_ptr, "/common/maneuver_overrides/overrides");
const auto maneuver_override_node_sequences_ptr = layout.GetBlockPtr<NodeID, true>(
memory_ptr, "/common/maneuver_overrides/node_sequences");
util::vector_view<extractor::StorageManeuverOverride> maneuver_overrides(
maneuver_overrides_ptr,
layout.GetBlockEntries("/common/maneuver_overrides/overrides"));
util::vector_view<NodeID> maneuver_override_node_sequences(
maneuver_override_node_sequences_ptr,
layout.GetBlockEntries("/common/maneuver_overrides/node_sequences"));
extractor::files::readManeuverOverrides(config.GetPath(".osrm.maneuver_overrides"),
maneuver_overrides,
maneuver_override_node_sequences);
}
extractor::files::readManeuverOverrides(config.GetPath(".osrm.maneuver_overrides"),
maneuver_overrides,
maneuver_override_node_sequences);
}
}
}

33
unit_tests/contractor/files.cpp
View File

@ -32,26 +32,33 @@ BOOST_AUTO_TEST_CASE(read_write_hsgr)
{true, true, true, true, true, true, true},
};
std::unordered_map<std::string, ContractedMetric> reference_metrics = {
{"duration", {std::move(reference_graph), std::move(reference_filters)}}};
TemporaryFile tmp{TEST_DATA_DIR "/read_write_hsgr_test.osrm.hsgr"};
contractor::files::writeGraph(tmp.path,
reference_checksum,
reference_graph,
reference_filters,
reference_connectivity_checksum);
contractor::files::writeGraph(
tmp.path, reference_checksum, reference_metrics, reference_connectivity_checksum);
unsigned checksum;
unsigned connectivity_checksum;
QueryGraph graph;
std::vector<std::vector<bool>> filters;
contractor::files::readGraph(tmp.path, checksum, graph, filters, connectivity_checksum);
std::unordered_map<std::string, ContractedMetric> metrics = {
{"duration", {}}
};
contractor::files::readGraph(tmp.path, checksum, metrics, connectivity_checksum);
BOOST_CHECK_EQUAL(checksum, reference_checksum);
BOOST_CHECK_EQUAL(connectivity_checksum, reference_connectivity_checksum);
BOOST_CHECK_EQUAL(filters.size(), reference_filters.size());
CHECK_EQUAL_COLLECTIONS(filters[0], reference_filters[0]);
CHECK_EQUAL_COLLECTIONS(filters[1], reference_filters[1]);
CHECK_EQUAL_COLLECTIONS(filters[2], reference_filters[2]);
CHECK_EQUAL_COLLECTIONS(filters[3], reference_filters[3]);
BOOST_CHECK_EQUAL(metrics["duration"].edge_filter.size(),
reference_metrics["duration"].edge_filter.size());
CHECK_EQUAL_COLLECTIONS(metrics["duration"].edge_filter[0],
reference_metrics["duration"].edge_filter[0]);
CHECK_EQUAL_COLLECTIONS(metrics["duration"].edge_filter[1],
reference_metrics["duration"].edge_filter[1]);
CHECK_EQUAL_COLLECTIONS(metrics["duration"].edge_filter[2],
reference_metrics["duration"].edge_filter[2]);
CHECK_EQUAL_COLLECTIONS(metrics["duration"].edge_filter[3],
reference_metrics["duration"].edge_filter[3]);
}
BOOST_AUTO_TEST_SUITE_END()