From 9b4a4fdd82df4a959b619fd3752fa6509976c6ed Mon Sep 17 00:00:00 2001 From: Michael Krasnyk Date: Sat, 21 Apr 2018 09:41:40 +0300 Subject: [PATCH] Add customizer::MultiLevelGraph --- include/customizer/edge_based_graph.hpp | 88 ++++++++++++++++++----- include/customizer/files.hpp | 33 +++++++++ include/customizer/serialization.hpp | 26 +++++++ include/partitioner/files.hpp | 64 ++++++++--------- include/partitioner/multi_level_graph.hpp | 15 ++++ include/partitioner/serialization.hpp | 20 ------ include/storage/view_factory.hpp | 9 ++- include/util/static_graph.hpp | 2 +- src/customize/customizer.cpp | 16 +++-- src/storage/storage.cpp | 2 +- 10 files changed, 194 insertions(+), 81 deletions(-) diff --git a/include/customizer/edge_based_graph.hpp b/include/customizer/edge_based_graph.hpp index 10a088dbd..555a2624b 100644 --- a/include/customizer/edge_based_graph.hpp +++ b/include/customizer/edge_based_graph.hpp @@ -16,28 +16,82 @@ namespace osrm namespace customizer { +// TODO: Change to turn_id only using EdgeBasedGraphEdgeData = partitioner::EdgeBasedGraphEdgeData; -struct MultiLevelEdgeBasedGraph - : public partitioner::MultiLevelGraph +template class MultiLevelGraph; + +namespace serialization { - using Base = - partitioner::MultiLevelGraph; - using Base::Base; +template +void read(storage::tar::FileReader &reader, + const std::string &name, + MultiLevelGraph &graph); + +template +void write(storage::tar::FileWriter &writer, + const std::string &name, + const MultiLevelGraph &graph); +} + +template +class MultiLevelGraph : public partitioner::MultiLevelGraph +{ + private: + using SuperT = partitioner::MultiLevelGraph; + using SuperC = partitioner::MultiLevelGraph; + template using Vector = util::ViewOrVector; + + public: + MultiLevelGraph() = default; + MultiLevelGraph(MultiLevelGraph &&) = default; + MultiLevelGraph(const MultiLevelGraph &) = default; + MultiLevelGraph &operator=(MultiLevelGraph &&) = default; + MultiLevelGraph &operator=(const MultiLevelGraph &) = default; + + // TODO: add constructor for EdgeBasedGraphEdgeData + MultiLevelGraph(SuperC &&graph, + Vector node_weights_, + Vector node_durations_) + : node_weights(std::move(node_weights_)), node_durations(std::move(node_durations_)) + { + std::tie(SuperT::node_array, + SuperT::edge_array, + SuperT::node_to_edge_offset, + SuperT::connectivity_checksum) = std::move(graph).data(); + // TODO: add EdgeArrayEntry shaving + } + + MultiLevelGraph(Vector node_array_, + Vector edge_array_, + Vector node_to_edge_offset_, + Vector node_weights_, + Vector node_durations_) + : SuperT(std::move(node_array_), std::move(edge_array_), std::move(node_to_edge_offset_)), + node_weights(std::move(node_weights_)), node_durations(std::move(node_durations_)) + { + // TODO: add EdgeArrayEntry shaving + } + + friend void + serialization::read(storage::tar::FileReader &reader, + const std::string &name, + MultiLevelGraph &graph); + friend void + serialization::write(storage::tar::FileWriter &writer, + const std::string &name, + const MultiLevelGraph &graph); + + protected: + Vector node_weights; + Vector node_durations; }; -struct MultiLevelEdgeBasedGraphView - : public partitioner::MultiLevelGraph -{ - using Base = partitioner::MultiLevelGraph; - using Base::Base; -}; - -struct StaticEdgeBasedGraphEdge : MultiLevelEdgeBasedGraph::InputEdge -{ - using Base = MultiLevelEdgeBasedGraph::InputEdge; - using Base::Base; -}; +using MultiLevelEdgeBasedGraph = + MultiLevelGraph; +using MultiLevelEdgeBasedGraphView = + MultiLevelGraph; } } diff --git a/include/customizer/files.hpp b/include/customizer/files.hpp index c89b5cb76..364a2572d 100644 --- a/include/customizer/files.hpp +++ b/include/customizer/files.hpp @@ -73,6 +73,39 @@ writeCellMetrics(const boost::filesystem::path &path, } } } + +// reads .osrm.mldgr file +template +inline void readGraph(const boost::filesystem::path &path, + MultiLevelGraphT &graph, + std::uint32_t &connectivity_checksum) +{ + static_assert(std::is_same::value || + std::is_same::value, + ""); + + storage::tar::FileReader reader{path, storage::tar::FileReader::VerifyFingerprint}; + + reader.ReadInto("/mld/connectivity_checksum", connectivity_checksum); + serialization::read(reader, "/mld/multilevelgraph", graph); +} + +// writes .osrm.mldgr file +template +inline void writeGraph(const boost::filesystem::path &path, + const MultiLevelGraphT &graph, + const std::uint32_t connectivity_checksum) +{ + static_assert(std::is_same::value || + std::is_same::value, + ""); + + storage::tar::FileWriter writer{path, storage::tar::FileWriter::GenerateFingerprint}; + + writer.WriteElementCount64("/mld/connectivity_checksum", 1); + writer.WriteFrom("/mld/connectivity_checksum", connectivity_checksum); + serialization::write(writer, "/mld/multilevelgraph", graph); +} } } } diff --git a/include/customizer/serialization.hpp b/include/customizer/serialization.hpp index 973e2517d..803f905a4 100644 --- a/include/customizer/serialization.hpp +++ b/include/customizer/serialization.hpp @@ -1,6 +1,8 @@ #ifndef OSRM_CUSTOMIZER_SERIALIZATION_HPP #define OSRM_CUSTOMIZER_SERIALIZATION_HPP +#include "customizer/edge_based_graph.hpp" + #include "partitioner/cell_storage.hpp" #include "storage/serialization.hpp" @@ -31,6 +33,30 @@ inline void write(storage::tar::FileWriter &writer, storage::serialization::write(writer, name + "/weights", metric.weights); storage::serialization::write(writer, name + "/durations", metric.durations); } + +template +inline void read(storage::tar::FileReader &reader, + const std::string &name, + MultiLevelGraph &graph) +{ + storage::serialization::read(reader, name + "/node_array", graph.node_array); + storage::serialization::read(reader, name + "/node_weights", graph.node_weights); + storage::serialization::read(reader, name + "/node_durations", graph.node_durations); + storage::serialization::read(reader, name + "/edge_array", graph.edge_array); + storage::serialization::read(reader, name + "/node_to_edge_offset", graph.node_to_edge_offset); +} + +template +inline void write(storage::tar::FileWriter &writer, + const std::string &name, + const MultiLevelGraph &graph) +{ + storage::serialization::write(writer, name + "/node_array", graph.node_array); + storage::serialization::write(writer, name + "/node_weights", graph.node_weights); + storage::serialization::write(writer, name + "/node_durations", graph.node_durations); + storage::serialization::write(writer, name + "/edge_array", graph.edge_array); + storage::serialization::write(writer, name + "/node_to_edge_offset", graph.node_to_edge_offset); +} } } } diff --git a/include/partitioner/files.hpp b/include/partitioner/files.hpp index 3a85a8fb0..30bd2942d 100644 --- a/include/partitioner/files.hpp +++ b/include/partitioner/files.hpp @@ -1,8 +1,6 @@ #ifndef OSRM_PARTITIONER_SERILIZATION_HPP #define OSRM_PARTITIONER_SERILIZATION_HPP -#include "customizer/edge_based_graph.hpp" - #include "partitioner/serialization.hpp" #include "storage/io.hpp" @@ -14,39 +12,6 @@ namespace partitioner namespace files { -// reads .osrm.mldgr file -template -inline void readGraph(const boost::filesystem::path &path, - MultiLevelGraphT &graph, - std::uint32_t &connectivity_checksum) -{ - static_assert(std::is_same::value || - std::is_same::value, - ""); - - storage::tar::FileReader reader{path, storage::tar::FileReader::VerifyFingerprint}; - - reader.ReadInto("/mld/connectivity_checksum", connectivity_checksum); - serialization::read(reader, "/mld/multilevelgraph", graph); -} - -// writes .osrm.mldgr file -template -inline void writeGraph(const boost::filesystem::path &path, - const MultiLevelGraphT &graph, - const std::uint32_t connectivity_checksum) -{ - static_assert(std::is_same::value || - std::is_same::value, - ""); - - storage::tar::FileWriter writer{path, storage::tar::FileWriter::GenerateFingerprint}; - - writer.WriteElementCount64("/mld/connectivity_checksum", 1); - writer.WriteFrom("/mld/connectivity_checksum", connectivity_checksum); - serialization::write(writer, "/mld/multilevelgraph", graph); -} - // read .osrm.partition file template inline void readPartition(const boost::filesystem::path &path, MultiLevelPartitionT &mlp) @@ -102,6 +67,35 @@ inline void writeCells(const boost::filesystem::path &path, CellStorageT &storag serialization::write(writer, "/mld/cellstorage", storage); } + +// reads .osrm.mldgr file +template +inline void readGraph(const boost::filesystem::path &path, + MultiLevelGraphT &graph, + std::uint32_t &connectivity_checksum) +{ + static_assert(std::is_same::value, ""); + + storage::tar::FileReader reader{path, storage::tar::FileReader::VerifyFingerprint}; + + reader.ReadInto("/mld/connectivity_checksum", connectivity_checksum); + serialization::read(reader, "/mld/multilevelgraph", graph); +} + +// writes .osrm.mldgr file +template +inline void writeGraph(const boost::filesystem::path &path, + const MultiLevelGraphT &graph, + const std::uint32_t connectivity_checksum) +{ + static_assert(std::is_same::value, ""); + + storage::tar::FileWriter writer{path, storage::tar::FileWriter::GenerateFingerprint}; + + writer.WriteElementCount64("/mld/connectivity_checksum", 1); + writer.WriteFrom("/mld/connectivity_checksum", connectivity_checksum); + serialization::write(writer, "/mld/multilevelgraph", graph); +} } } } diff --git a/include/partitioner/multi_level_graph.hpp b/include/partitioner/multi_level_graph.hpp index b652589f6..f712e6d18 100644 --- a/include/partitioner/multi_level_graph.hpp +++ b/include/partitioner/multi_level_graph.hpp @@ -1,6 +1,7 @@ #ifndef OSRM_PARTITIONER_MULTI_LEVEL_GRAPH_HPP #define OSRM_PARTITIONER_MULTI_LEVEL_GRAPH_HPP +#include "partitioner/edge_based_graph.hpp" #include "partitioner/multi_level_partition.hpp" #include "storage/shared_memory_ownership.hpp" @@ -42,6 +43,8 @@ class MultiLevelGraph : public util::StaticGraph template using Vector = util::ViewOrVector; public: + using SuperT::SuperT; + // We limit each node to have 255 edges // this is very generous, we could probably pack this using EdgeOffset = std::uint8_t; @@ -146,6 +149,14 @@ class MultiLevelGraph : public util::StaticGraph return max_border_node_id; } + auto data() && + { + return std::make_tuple(std::move(SuperT::node_array), + std::move(SuperT::edge_array), + std::move(node_to_edge_offset), + connectivity_checksum); + } + private: template auto GetHighestBorderLevel(const MultiLevelPartition &mlp, const ContainerT &edges) const @@ -218,9 +229,13 @@ class MultiLevelGraph : public util::StaticGraph const std::string &name, const MultiLevelGraph &graph); + protected: Vector node_to_edge_offset; std::uint32_t connectivity_checksum; }; + +using MultiLevelEdgeBasedGraph = + MultiLevelGraph; } } diff --git a/include/partitioner/serialization.hpp b/include/partitioner/serialization.hpp index b8d57bb6d..6f43c02ea 100644 --- a/include/partitioner/serialization.hpp +++ b/include/partitioner/serialization.hpp @@ -19,26 +19,6 @@ namespace partitioner namespace serialization { -template -inline void read(storage::tar::FileReader &reader, - const std::string &name, - MultiLevelGraph &graph) -{ - storage::serialization::read(reader, name + "/node_array", graph.node_array); - storage::serialization::read(reader, name + "/edge_array", graph.edge_array); - storage::serialization::read(reader, name + "/node_to_edge_offset", graph.node_to_edge_offset); -} - -template -inline void write(storage::tar::FileWriter &writer, - const std::string &name, - const MultiLevelGraph &graph) -{ - storage::serialization::write(writer, name + "/node_array", graph.node_array); - storage::serialization::write(writer, name + "/edge_array", graph.edge_array); - storage::serialization::write(writer, name + "/node_to_edge_offset", graph.node_to_edge_offset); -} - template inline void read(storage::tar::FileReader &reader, const std::string &name, diff --git a/include/storage/view_factory.hpp b/include/storage/view_factory.hpp index cf6839a05..cb4d58302 100644 --- a/include/storage/view_factory.hpp +++ b/include/storage/view_factory.hpp @@ -330,9 +330,14 @@ inline auto make_multi_level_graph_view(const SharedDataIndex &index, const std: index, name + "/edge_array"); auto node_to_offset = make_vector_view( index, name + "/node_to_edge_offset"); + auto node_weights = make_vector_view(index, name + "/node_weights"); + auto node_durations = make_vector_view(index, name + "/node_durations"); - return customizer::MultiLevelEdgeBasedGraphView( - std::move(node_list), std::move(edge_list), std::move(node_to_offset)); + return customizer::MultiLevelEdgeBasedGraphView(std::move(node_list), + std::move(edge_list), + std::move(node_to_offset), + std::move(node_weights), + std::move(node_durations)); } inline auto make_maneuver_overrides_views(const SharedDataIndex &index, const std::string &name) diff --git a/include/util/static_graph.hpp b/include/util/static_graph.hpp index 433a830c1..6092dd955 100644 --- a/include/util/static_graph.hpp +++ b/include/util/static_graph.hpp @@ -312,7 +312,7 @@ class StaticGraph }); } - // private: + protected: NodeIterator number_of_nodes; EdgeIterator number_of_edges; diff --git a/src/customize/customizer.cpp b/src/customize/customizer.cpp index 50007e4f4..8a01c4e8d 100644 --- a/src/customize/customizer.cpp +++ b/src/customize/customizer.cpp @@ -84,14 +84,15 @@ auto LoadAndUpdateEdgeExpandedGraph(const CustomizationConfig &config, updater.LoadAndUpdateEdgeExpandedGraph(); auto directed = partitioner::splitBidirectionalEdges(edge_based_edge_list); - auto tidied = - partitioner::prepareEdgesForUsageInGraph(std::move(directed)); - auto edge_based_graph = customizer::MultiLevelEdgeBasedGraph(mlp, num_nodes, std::move(tidied)); + auto tidied = partitioner::prepareEdgesForUsageInGraph< + typename partitioner::MultiLevelEdgeBasedGraph::InputEdge>(std::move(directed)); + auto edge_based_graph = + partitioner::MultiLevelEdgeBasedGraph(mlp, num_nodes, std::move(tidied)); return edge_based_graph; } -std::vector customizeFilteredMetrics(const MultiLevelEdgeBasedGraph &graph, +std::vector customizeFilteredMetrics(const partitioner::MultiLevelEdgeBasedGraph &graph, const partitioner::CellStorage &storage, const CellCustomizer &customizer, const std::vector> &node_filters) @@ -157,7 +158,12 @@ int Customizer::Run(const CustomizationConfig &config) util::Log() << "MLD customization writing took " << TIMER_SEC(writing_mld_data) << " seconds"; TIMER_START(writing_graph); - partitioner::files::writeGraph(config.GetPath(".osrm.mldgr"), graph, connectivity_checksum); + std::vector node_weights; + std::vector node_durations; + MultiLevelEdgeBasedGraph shaved_graph{ + std::move(graph), std::move(node_weights), std::move(node_durations)}; + customizer::files::writeGraph( + config.GetPath(".osrm.mldgr"), shaved_graph, connectivity_checksum); TIMER_STOP(writing_graph); util::Log() << "Graph writing took " << TIMER_SEC(writing_graph) << " seconds"; diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 7ae693739..bcdf03252 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -554,7 +554,7 @@ void Storage::PopulateUpdatableData(const SharedDataIndex &index) { auto graph_view = make_multi_level_graph_view(index, "/mld/multilevelgraph"); std::uint32_t graph_connectivity_checksum = 0; - partitioner::files::readGraph( + customizer::files::readGraph( config.GetPath(".osrm.mldgr"), graph_view, graph_connectivity_checksum); auto turns_connectivity_checksum =