diff --git a/include/extractor/compressed_node_based_graph_edge.hpp b/include/extractor/compressed_node_based_graph_edge.hpp new file mode 100644 index 000000000..fc74b862c --- /dev/null +++ b/include/extractor/compressed_node_based_graph_edge.hpp @@ -0,0 +1,21 @@ +#ifndef OSRM_EXTRACTOR_COMPRESSED_NODE_BASED_GRAPH_EDGE_HPP +#define OSRM_EXTRACTOR_COMPRESSED_NODE_BASED_GRAPH_EDGE_HPP + +#include "util/typedefs.hpp" + +namespace osrm +{ +namespace extractor +{ + +// We encode the cnbg graph only using its topology as edge list +struct CompressedNodeBasedGraphEdge +{ + NodeID source; + NodeID target; +}; + +} +} + +#endif // OSRM_EXTRACTOR_COMPRESSED_NODE_BASED_GRAPH_EDGE_HPP diff --git a/include/extractor/extractor.hpp b/include/extractor/extractor.hpp index fc1df6382..0427cfaba 100644 --- a/include/extractor/extractor.hpp +++ b/include/extractor/extractor.hpp @@ -99,11 +99,6 @@ class Extractor const std::vector &coordinates); std::shared_ptr LoadRestrictionMap(); - // Writes compressed node based graph and its embedding into a file for osrm-partition to use. - static void WriteCompressedNodeBasedGraph(const std::string &path, - const util::NodeBasedDynamicGraph &graph, - const std::vector &coordiantes); - void WriteConditionalRestrictions( const std::string &path, std::vector &conditional_turn_restrictions); diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 0d0820cff..57efba7a7 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -94,6 +94,7 @@ void writeEdgeBasedGraph(const boost::filesystem::path &path, writer.WriteFrom("/common/connectivity_checksum", connectivity_checksum); } +// reads .osrm.ebg file template void readEdgeBasedGraph(const boost::filesystem::path &path, EdgeID &number_of_edge_based_nodes, @@ -125,6 +126,19 @@ inline void readNodes(const boost::filesystem::path &path, util::serialization::read(reader, "/common/osm_node_ids", osm_node_ids); } +// reads only coordinates from .osrm.nbg_nodes +template +inline void readNodeCoordinates(const boost::filesystem::path &path, + CoordinatesT &coordinates) +{ + static_assert(std::is_same::value, ""); + + const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; + storage::tar::FileReader reader{path, fingerprint}; + + storage::serialization::read(reader, "/common/coordinates", coordinates); +} + // writes .osrm.nbg_nodes template inline void writeNodes(const boost::filesystem::path &path, @@ -476,6 +490,24 @@ void readRamIndex(const boost::filesystem::path &path, RTreeT &rtree) util::serialization::read(reader, "/common/rtree", rtree); } + +template +void writeCompressedNodeBasedGraph(const boost::filesystem::path &path, const EdgeListT &edge_list) +{ + const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; + storage::tar::FileWriter writer{path, fingerprint}; + + storage::serialization::write(writer, "/extractor/cnbg", edge_list); +} + +template +void readCompressedNodeBasedGraph(const boost::filesystem::path &path, EdgeListT &edge_list) +{ + const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; + storage::tar::FileReader reader{path, fingerprint}; + + storage::serialization::read(reader, "/extractor/cnbg", edge_list); +} } } } diff --git a/include/partitioner/compressed_node_based_graph_reader.hpp b/include/partitioner/compressed_node_based_graph_reader.hpp deleted file mode 100644 index d477f21b8..000000000 --- a/include/partitioner/compressed_node_based_graph_reader.hpp +++ /dev/null @@ -1,60 +0,0 @@ -#ifndef OSRM_PARTITIONER_COMPRESSED_NODE_BASED_GRAPH_READER_HPP -#define OSRM_PARTITIONER_COMPRESSED_NODE_BASED_GRAPH_READER_HPP - -#include "storage/io.hpp" -#include "util/coordinate.hpp" -#include "util/typedefs.hpp" - -#include -#include - -namespace osrm -{ -namespace partitioner -{ - -struct CompressedNodeBasedGraphEdge -{ - NodeID source; - NodeID target; -}; - -struct CompressedNodeBasedGraph -{ - CompressedNodeBasedGraph(storage::io::FileReader &reader) - { - // Reads: | Fingerprint | #e | #n | edges | coordinates | - // - uint64: number of edges (from, to) pairs - // - uint64: number of nodes and therefore also coordinates - // - (uint32_t, uint32_t): num_edges * edges - // - (int32_t, int32_t: num_nodes * coordinates (lon, lat) - // - // Gets written in Extractor::WriteCompressedNodeBasedGraph - - const auto num_edges = reader.ReadElementCount64(); - const auto num_nodes = reader.ReadElementCount64(); - - edges.resize(num_edges); - coordinates.resize(num_nodes); - - reader.ReadInto(edges); - reader.ReadInto(coordinates); - } - - std::vector edges; - std::vector coordinates; -}; - -inline CompressedNodeBasedGraph LoadCompressedNodeBasedGraph(const std::string &path) -{ - const auto fingerprint = storage::io::FileReader::VerifyFingerprint; - storage::io::FileReader reader(path, fingerprint); - - CompressedNodeBasedGraph graph{reader}; - return graph; -} - -} // ns partition -} // ns osrm - -#endif diff --git a/include/partitioner/partitioner_config.hpp b/include/partitioner/partitioner_config.hpp index 83b8e7782..31423c919 100644 --- a/include/partitioner/partitioner_config.hpp +++ b/include/partitioner/partitioner_config.hpp @@ -21,6 +21,7 @@ struct PartitionerConfig final : storage::IOConfig {".osrm.ebg", ".osrm.cnbg", ".osrm.cnbg_to_ebg", + ".osrm.nbg_nodes", ".osrm.partition", ".osrm.cells", ".osrm.maneuver_overrides"}), diff --git a/src/benchmarks/static_rtree.cpp b/src/benchmarks/static_rtree.cpp index 05ca8bd5c..60fbb255f 100644 --- a/src/benchmarks/static_rtree.cpp +++ b/src/benchmarks/static_rtree.cpp @@ -33,15 +33,6 @@ constexpr int32_t WORLD_MAX_LON = 180 * COORDINATE_PRECISION; using RTreeLeaf = extractor::EdgeBasedNodeSegment; using BenchStaticRTree = util::StaticRTree; -std::vector loadCoordinates(const boost::filesystem::path &nodes_file) -{ - std::vector coords; - extractor::PackedOSMIDs nodes; - extractor::files::readNodes(nodes_file, coords, nodes); - - return coords; -} - template void benchmarkQuery(const std::vector &queries, const std::string &name, @@ -99,7 +90,8 @@ int main(int argc, char **argv) const char *file_path = argv[2]; const char *nodes_path = argv[3]; - auto coords = osrm::benchmarks::loadCoordinates(nodes_path); + std::vector coords; + osrm::extractor::files::readNodeCoordinates(nodes_path, coords); osrm::benchmarks::BenchStaticRTree rtree(file_path, coords); osrm::extractor::files::readRamIndex(ram_path, rtree); diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index e6c70d137..d9f3f4ba4 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -14,6 +14,7 @@ #include "extractor/restriction_parser.hpp" #include "extractor/scripting_environment.hpp" #include "extractor/name_table.hpp" +#include "extractor/compressed_node_based_graph_edge.hpp" #include "guidance/files.hpp" #include "guidance/guidance_processing.hpp" @@ -168,6 +169,26 @@ void SetExcludableClasses(const ExtractorCallbacks::ClassesMap &classes_map, } } } + +std::vector toEdgeList(const util::NodeBasedDynamicGraph& graph) +{ + std::vector edges; + edges.reserve(graph.GetNumberOfEdges()); + + // For all nodes iterate over its edges and dump (from, to) pairs + for (const NodeID from_node : util::irange(0u, graph.GetNumberOfNodes())) + { + for (const EdgeID edge : graph.GetAdjacentEdgeRange(from_node)) + { + const auto to_node = graph.GetTarget(edge); + + edges.push_back({from_node, to_node}); + } + } + + return edges; +} + } /** @@ -267,10 +288,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) compressed_node_based_graph_writing.wait(); }; - compressed_node_based_graph_writing = std::async(std::launch::async, [&] { - WriteCompressedNodeBasedGraph( - config.GetPath(".osrm.cnbg").string(), node_based_graph, coordinates); - }); + files::writeCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg").string(), toEdgeList(node_based_graph)); node_based_graph_factory.GetCompressedEdges().PrintStatistics(); @@ -814,48 +832,6 @@ void Extractor::BuildRTree(std::vector edge_based_node_seg util::Log() << "finished r-tree construction in " << TIMER_SEC(construction) << " seconds"; } -void Extractor::WriteCompressedNodeBasedGraph(const std::string &path, - const util::NodeBasedDynamicGraph &graph, - const std::vector &coordinates) -{ - const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; - - storage::io::FileWriter writer{path, fingerprint}; - - // Writes: | Fingerprint | #e | #n | edges | coordinates | - // - uint64: number of edges (from, to) pairs - // - uint64: number of nodes and therefore also coordinates - // - (uint32_t, uint32_t): num_edges * edges - // - (int32_t, int32_t: num_nodes * coordinates (lon, lat) - - const auto num_edges = graph.GetNumberOfEdges(); - const auto num_nodes = graph.GetNumberOfNodes(); - - BOOST_ASSERT_MSG(num_nodes == coordinates.size(), "graph and embedding out of sync"); - - writer.WriteElementCount64(num_edges); - writer.WriteElementCount64(num_nodes); - - // For all nodes iterate over its edges and dump (from, to) pairs - for (const NodeID from_node : util::irange(0u, num_nodes)) - { - for (const EdgeID edge : graph.GetAdjacentEdgeRange(from_node)) - { - const auto to_node = graph.GetTarget(edge); - - writer.WriteFrom(from_node); - writer.WriteFrom(to_node); - } - } - - // FIXME this is unneccesary: We have this data - for (const auto &qnode : coordinates) - { - writer.WriteFrom(qnode.lon); - writer.WriteFrom(qnode.lat); - } -} - template auto convertIDMapToVector(const Map &map) { std::vector result(map.size()); diff --git a/src/partitioner/partitioner.cpp b/src/partitioner/partitioner.cpp index 87294ba6e..2f960cfd0 100644 --- a/src/partitioner/partitioner.cpp +++ b/src/partitioner/partitioner.cpp @@ -2,7 +2,6 @@ #include "partitioner/bisection_graph.hpp" #include "partitioner/bisection_to_partition.hpp" #include "partitioner/cell_storage.hpp" -#include "partitioner/compressed_node_based_graph_reader.hpp" #include "partitioner/edge_based_graph_reader.hpp" #include "partitioner/files.hpp" #include "partitioner/multi_level_partition.hpp" @@ -10,6 +9,7 @@ #include "partitioner/remove_unconnected.hpp" #include "partitioner/renumber.hpp" +#include "extractor/compressed_node_based_graph_edge.hpp" #include "extractor/files.hpp" #include "util/coordinate.hpp" @@ -40,19 +40,17 @@ namespace partitioner { auto getGraphBisection(const PartitionerConfig &config) { - auto compressed_node_based_graph = - LoadCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg").string()); + std::vector edges; + extractor::files::readCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg"), edges); + groupEdgesBySource(begin(edges), end(edges)); - util::Log() << "Loaded compressed node based graph: " - << compressed_node_based_graph.edges.size() << " edges, " - << compressed_node_based_graph.coordinates.size() << " nodes"; + std::vector coordinates; + extractor::files::readNodeCoordinates(config.GetPath(".osrm.nbg_nodes"), coordinates); - groupEdgesBySource(begin(compressed_node_based_graph.edges), - end(compressed_node_based_graph.edges)); + util::Log() << "Loaded compressed node based graph: " << edges.size() << " edges, " + << coordinates.size() << " nodes"; - auto graph = - makeBisectionGraph(compressed_node_based_graph.coordinates, - adaptToBisectionEdge(std::move(compressed_node_based_graph.edges))); + auto graph = makeBisectionGraph(coordinates, adaptToBisectionEdge(std::move(edges))); util::Log() << " running partition: " << config.max_cell_sizes.front() << " " << config.balance << " " << config.boundary_factor << " " << config.num_optimizing_cuts << " "