From c334d11e950fd851ed61a8712e26ba15043c412b Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 27 Mar 2018 09:44:13 +0000 Subject: [PATCH] Refactor metric storage --- include/contractor/contracted_metric.hpp | 26 ++ include/contractor/files.hpp | 59 +-- include/contractor/serialization.hpp | 53 +++ include/customizer/files.hpp | 42 +- include/engine/algorithm.hpp | 5 + .../contiguous_internalmem_datafacade.hpp | 57 +-- include/engine/datafacade_factory.hpp | 35 +- include/engine/engine.hpp | 47 ++- include/storage/serialization.hpp | 131 +++--- include/storage/shared_datatype.hpp | 2 - src/contractor/contractor.cpp | 8 +- src/customize/customizer.cpp | 5 +- src/storage/storage.cpp | 372 +++++++++--------- unit_tests/contractor/files.cpp | 33 +- 14 files changed, 505 insertions(+), 370 deletions(-) create mode 100644 include/contractor/contracted_metric.hpp create mode 100644 include/contractor/serialization.hpp diff --git a/include/contractor/contracted_metric.hpp b/include/contractor/contracted_metric.hpp new file mode 100644 index 000000000..e8592b32a --- /dev/null +++ b/include/contractor/contracted_metric.hpp @@ -0,0 +1,26 @@ +#ifndef OSMR_CONTRACTOR_CONTRACTED_METRIC_HPP +#define OSMR_CONTRACTOR_CONTRACTED_METRIC_HPP + +#include "contractor/query_graph.hpp" + +namespace osrm +{ +namespace contractor +{ + +namespace detail +{ +template +struct ContractedMetric +{ + detail::QueryGraph graph; + std::vector> edge_filter; +}; +} + +using ContractedMetric = detail::ContractedMetric; +using ContractedMetricView = detail::ContractedMetric; +} +} + +#endif diff --git a/include/contractor/files.hpp b/include/contractor/files.hpp index 39a15acaa..b7014f9ab 100644 --- a/include/contractor/files.hpp +++ b/include/contractor/files.hpp @@ -1,12 +1,9 @@ #ifndef OSRM_CONTRACTOR_FILES_HPP #define OSRM_CONTRACTOR_FILES_HPP -#include "contractor/query_graph.hpp" +#include "contractor/serialization.hpp" -#include "util/serialization.hpp" - -#include "storage/serialization.hpp" -#include "storage/tar.hpp" +#include namespace osrm { @@ -15,67 +12,51 @@ namespace contractor namespace files { // reads .osrm.hsgr file -template +template inline void readGraph(const boost::filesystem::path &path, unsigned &checksum, - QueryGraphT &graph, - std::vector &edge_filter, + std::unordered_map &metrics, std::uint32_t &connectivity_checksum) { - static_assert(std::is_same::value || - std::is_same::value, - "graph must be of type QueryGraph<>"); - static_assert(std::is_same>::value || - std::is_same>::value, - "edge_filter must be a container of vector or vector_view"); + static_assert(std::is_same::value || + std::is_same::value, + "metric must be of type ContractedMetric<>"); const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; storage::tar::FileReader reader{path, fingerprint}; reader.ReadInto("/ch/checksum", checksum); - util::serialization::read(reader, "/ch/contracted_graph", graph); + reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum); - auto count = reader.ReadElementCount64("/ch/edge_filter"); - edge_filter.resize(count); - for (const auto index : util::irange(0, count)) + for (auto &pair : metrics) { - storage::serialization::read( - reader, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]); + serialization::read(reader, "/ch/metrics/" + pair.first, pair.second); } - reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum); } // writes .osrm.hsgr file -template +template inline void writeGraph(const boost::filesystem::path &path, unsigned checksum, - const QueryGraphT &graph, - const std::vector &edge_filter, + const std::unordered_map &metrics, const std::uint32_t connectivity_checksum) { - static_assert(std::is_same::value || - std::is_same::value, - "graph must be of type QueryGraph<>"); - static_assert(std::is_same>::value || - std::is_same>::value, - "edge_filter must be a container of vector or vector_view"); + static_assert(std::is_same::value || + std::is_same::value, + "metric must be of type ContractedMetric<>"); const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; storage::tar::FileWriter writer{path, fingerprint}; writer.WriteElementCount64("/ch/checksum", 1); writer.WriteFrom("/ch/checksum", checksum); - util::serialization::write(writer, "/ch/contracted_graph", graph); - - writer.WriteElementCount64("/ch/edge_filter", edge_filter.size()); - for (const auto index : util::irange(0, edge_filter.size())) - { - storage::serialization::write( - writer, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]); - } - writer.WriteElementCount64("/ch/connectivity_checksum", 1); writer.WriteFrom("/ch/connectivity_checksum", connectivity_checksum); + + for (const auto &pair : metrics) + { + serialization::write(writer, "/ch/metrics/" + pair.first, pair.second); + } } } } diff --git a/include/contractor/serialization.hpp b/include/contractor/serialization.hpp new file mode 100644 index 000000000..aa8659274 --- /dev/null +++ b/include/contractor/serialization.hpp @@ -0,0 +1,53 @@ +#ifndef OSRM_CONTRACTOR_SERIALIZATION_HPP +#define OSRM_CONTRACTOR_SERIALIZATION_HPP + +#include "contractor/contracted_metric.hpp" + +#include "util/serialization.hpp" + +#include "storage/serialization.hpp" +#include "storage/tar.hpp" + +namespace osrm +{ +namespace contractor +{ +namespace serialization +{ + +template +void write(storage::tar::FileWriter &writer, + const std::string &name, + const detail::ContractedMetric &metric) +{ + util::serialization::write(writer, name + "/contracted_graph", metric.graph); + + writer.WriteElementCount64(name + "/exclude", metric.edge_filter.size()); + for (const auto index : util::irange(0, metric.edge_filter.size())) + { + storage::serialization::write(writer, + name + "/exclude/" + std::to_string(index) + "/edge_filter", + metric.edge_filter[index]); + } +} + +template +void read(storage::tar::FileReader &reader, + const std::string &name, + detail::ContractedMetric &metric) +{ + util::serialization::read(reader, name + "/contracted_graph", metric.graph); + + metric.edge_filter.resize(reader.ReadElementCount64(name + "/exclude")); + for (const auto index : util::irange(0, metric.edge_filter.size())) + { + storage::serialization::read(reader, + name + "/exclude/" + std::to_string(index) + "/edge_filter", + metric.edge_filter[index]); + } +} +} +} +} + +#endif diff --git a/include/customizer/files.hpp b/include/customizer/files.hpp index 098752f9c..8b690ecd2 100644 --- a/include/customizer/files.hpp +++ b/include/customizer/files.hpp @@ -7,6 +7,8 @@ #include "util/integer_range.hpp" +#include + namespace osrm { namespace customizer @@ -16,7 +18,7 @@ namespace files // reads .osrm.cell_metrics file template -inline void readCellMetrics(const boost::filesystem::path &path, std::vector &metrics) +inline void readCellMetrics(const boost::filesystem::path &path, std::unordered_map> &metrics) { static_assert(std::is_same::value || std::is_same::value, @@ -25,20 +27,27 @@ inline void readCellMetrics(const boost::filesystem::path &path, std::vector inline void writeCellMetrics(const boost::filesystem::path &path, - const std::vector &metrics) + const std::unordered_map> &metrics) { static_assert(std::is_same::value || std::is_same::value, @@ -47,12 +56,19 @@ inline void writeCellMetrics(const boost::filesystem::path &path, const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; storage::tar::FileWriter writer{path, fingerprint}; - writer.WriteElementCount64("/mld/metrics", metrics.size()); - - auto id = 0; - for (const auto &metric : metrics) + for (const auto& pair : metrics) { - serialization::write(writer, "/mld/metrics/" + std::to_string(id++), metric); + const auto& metric_name = pair.first; + const auto& metric_exclude_classes = pair.second; + + auto prefix = "/mld/metrics/" + metric_name + "/exclude"; + writer.WriteElementCount64(prefix, metric_exclude_classes.size()); + + auto id = 0; + for (auto &exclude_metric : metric_exclude_classes) + { + serialization::write(writer, prefix + "/" + std::to_string(id++), exclude_metric); + } } } } diff --git a/include/engine/algorithm.hpp b/include/engine/algorithm.hpp index 751ef6b9b..8f48b3730 100644 --- a/include/engine/algorithm.hpp +++ b/include/engine/algorithm.hpp @@ -30,6 +30,11 @@ template const char *name(); template <> inline const char *name() { return "CH"; } template <> inline const char *name() { return "MLD"; } +// Algorithm identifier +template const char *identifier(); +template <> inline const char *identifier() { return "ch"; } +template <> inline const char *identifier() { return "mld"; } + template struct HasAlternativePathSearch final : std::false_type { }; diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp index 96b424cd0..65f450459 100644 --- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp +++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp @@ -83,23 +83,26 @@ class ContiguousInternalMemoryAlgorithmDataFacade : public datafacade::Algor void InitializeGraphPointer(const storage::DataLayout &data_layout, char *memory_block, + const std::string& metric_name, const std::size_t exclude_index) { + const std::string metric_prefix = "/ch/metrics/" + metric_name; auto graph_nodes_ptr = - data_layout.GetBlockPtr(memory_block, "/ch/contracted_graph/node_array"); + data_layout.GetBlockPtr(memory_block, metric_prefix + "/contracted_graph/node_array"); auto graph_edges_ptr = - data_layout.GetBlockPtr(memory_block, "/ch/contracted_graph/edge_array"); + data_layout.GetBlockPtr(memory_block, metric_prefix + "/contracted_graph/edge_array"); - auto filter_block_id = "/ch/edge_filter/" + std::to_string(exclude_index); + auto exclude_prefix = metric_prefix + "/exclude/" + std::to_string(exclude_index); + auto filter_block_id = exclude_prefix + "/edge_filter"; auto edge_filter_ptr = data_layout.GetBlockPtr::Word>(memory_block, filter_block_id); util::vector_view node_list( - graph_nodes_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/node_array")); + graph_nodes_ptr, data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/node_array")); util::vector_view edge_list( - graph_edges_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/edge_array")); + graph_edges_ptr, data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/edge_array")); util::vector_view edge_filter(edge_filter_ptr, data_layout.GetBlockEntries(filter_block_id)); @@ -108,17 +111,18 @@ class ContiguousInternalMemoryAlgorithmDataFacade : public datafacade::Algor public: ContiguousInternalMemoryAlgorithmDataFacade( - std::shared_ptr allocator_, std::size_t exclude_index) + std::shared_ptr allocator_, const std::string& metric_name, std::size_t exclude_index) : allocator(std::move(allocator_)) { - InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index); + InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index); } void InitializeInternalPointers(const storage::DataLayout &data_layout, char *memory_block, + const std::string& metric_name, const std::size_t exclude_index) { - InitializeGraphPointer(data_layout, memory_block, exclude_index); + InitializeGraphPointer(data_layout, memory_block, metric_name, exclude_index); } // search graph access @@ -225,8 +229,11 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade void InitializeProfilePropertiesPointer(const storage::DataLayout &data_layout, char *memory_block, + const std::string& metric_name, const std::size_t exclude_index) { + // TODO: For multi-metric support we need to have separate exclude classes per metric + (void) metric_name; m_profile_properties = data_layout.GetBlockPtr( memory_block, "/common/properties"); @@ -528,6 +535,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade void InitializeInternalPointers(const storage::DataLayout &data_layout, char *memory_block, + const std::string& metric_name, const std::size_t exclude_index) { InitializeChecksumPointer(data_layout, memory_block); @@ -538,7 +546,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade InitializeGeometryPointers(data_layout, memory_block); InitializeNamePointers(data_layout, memory_block); InitializeTurnLaneDescriptionsPointers(data_layout, memory_block); - InitializeProfilePropertiesPointer(data_layout, memory_block, exclude_index); + InitializeProfilePropertiesPointer(data_layout, memory_block, metric_name, exclude_index); InitializeRTreePointers(data_layout, memory_block); InitializeIntersectionClassPointers(data_layout, memory_block); InitializeManeuverOverridePointers(data_layout, memory_block); @@ -548,10 +556,11 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade // allows switching between process_memory/shared_memory datafacade, based on the type of // allocator ContiguousInternalMemoryDataFacadeBase(std::shared_ptr allocator_, + const std::string& metric_name, const std::size_t exclude_index) : allocator(std::move(allocator_)) { - InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index); + InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index); } // node and edge information access @@ -959,9 +968,10 @@ class ContiguousInternalMemoryDataFacade { public: ContiguousInternalMemoryDataFacade(std::shared_ptr allocator, + const std::string& metric_name, const std::size_t exclude_index) - : ContiguousInternalMemoryDataFacadeBase(allocator, exclude_index), - ContiguousInternalMemoryAlgorithmDataFacade(allocator, exclude_index) + : ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index), + ContiguousInternalMemoryAlgorithmDataFacade(allocator, metric_name, exclude_index) { } @@ -981,14 +991,16 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade : public Algo void InitializeInternalPointers(const storage::DataLayout &data_layout, char *memory_block, + const std::string& metric_name, const std::size_t exclude_index) { - InitializeMLDDataPointers(data_layout, memory_block, exclude_index); + InitializeMLDDataPointers(data_layout, memory_block, metric_name, exclude_index); InitializeGraphPointer(data_layout, memory_block); } void InitializeMLDDataPointers(const storage::DataLayout &data_layout, char *memory_block, + const std::string& metric_name, const std::size_t exclude_index) { if (data_layout.GetBlockSize("/mld/multilevelpartition/partition") > 0) @@ -1016,9 +1028,9 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade : public Algo partitioner::MultiLevelPartitionView{level_data, partition, cell_to_children}; } - const auto weights_block_id = "/mld/metrics/" + std::to_string(exclude_index) + "/weights"; - const auto durations_block_id = - "/mld/metrics/" + std::to_string(exclude_index) + "/durations"; + const auto exclude_prefix = "/mld/metrics/" + metric_name + "/exclude/" + std::to_string(exclude_index); + const auto weights_block_id = exclude_prefix + "/weights"; + const auto durations_block_id = exclude_prefix + "/durations"; if (data_layout.GetBlockSize(weights_block_id) > 0) { @@ -1026,8 +1038,8 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade : public Algo data_layout.GetBlockPtr(memory_block, weights_block_id); auto mld_cell_durations_ptr = data_layout.GetBlockPtr(memory_block, durations_block_id); - auto weight_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/weights"); - auto duration_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/durations"); + auto weight_entries_count = data_layout.GetBlockEntries(weights_block_id); + auto duration_entries_count = data_layout.GetBlockEntries(durations_block_id); BOOST_ASSERT(weight_entries_count == duration_entries_count); util::vector_view weights(mld_cell_weights_ptr, weight_entries_count); util::vector_view durations(mld_cell_durations_ptr, @@ -1099,10 +1111,10 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade : public Algo public: ContiguousInternalMemoryAlgorithmDataFacade( - std::shared_ptr allocator_, const std::size_t exclude_index) + std::shared_ptr allocator_, const std::string& metric_name, const std::size_t exclude_index) : allocator(std::move(allocator_)) { - InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index); + InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index); } const partitioner::MultiLevelPartitionView &GetMultiLevelPartition() const override @@ -1156,9 +1168,10 @@ class ContiguousInternalMemoryDataFacade final private: public: ContiguousInternalMemoryDataFacade(std::shared_ptr allocator, + const std::string& metric_name, const std::size_t exclude_index) - : ContiguousInternalMemoryDataFacadeBase(allocator, exclude_index), - ContiguousInternalMemoryAlgorithmDataFacade(allocator, exclude_index) + : ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index), + ContiguousInternalMemoryAlgorithmDataFacade(allocator, metric_name, exclude_index) { } diff --git a/include/engine/datafacade_factory.hpp b/include/engine/datafacade_factory.hpp index 85de49bf5..bb313db90 100644 --- a/include/engine/datafacade_factory.hpp +++ b/include/engine/datafacade_factory.hpp @@ -33,6 +33,7 @@ template