Port .cnbg file to tar format

include/extractor/compressed_node_based_graph_edge.hpp

@@ -0,0 +1,21 @@
+#ifndef OSRM_EXTRACTOR_COMPRESSED_NODE_BASED_GRAPH_EDGE_HPP
+#define OSRM_EXTRACTOR_COMPRESSED_NODE_BASED_GRAPH_EDGE_HPP
+
+#include "util/typedefs.hpp"
+
+namespace osrm
+{
+namespace extractor
+{
+
+// We encode the cnbg graph only using its topology as edge list
+struct CompressedNodeBasedGraphEdge
+{
+    NodeID source;
+    NodeID target;
+};
+
+}
+}
+
+#endif // OSRM_EXTRACTOR_COMPRESSED_NODE_BASED_GRAPH_EDGE_HPP

include/extractor/extractor.hpp

@@ -99,11 +99,6 @@ class Extractor
                     const std::vector<util::Coordinate> &coordinates);
     std::shared_ptr<RestrictionMap> LoadRestrictionMap();
 
-    // Writes compressed node based graph and its embedding into a file for osrm-partition to use.
-    static void WriteCompressedNodeBasedGraph(const std::string &path,
-                                              const util::NodeBasedDynamicGraph &graph,
-                                              const std::vector<util::Coordinate> &coordiantes);
-
     void WriteConditionalRestrictions(
         const std::string &path,
         std::vector<ConditionalTurnRestriction> &conditional_turn_restrictions);

include/extractor/files.hpp

@@ -94,6 +94,7 @@ void writeEdgeBasedGraph(const boost::filesystem::path &path,
     writer.WriteFrom("/common/connectivity_checksum", connectivity_checksum);
 }
 
+// reads .osrm.ebg file
 template <typename EdgeBasedEdgeVector>
 void readEdgeBasedGraph(const boost::filesystem::path &path,
                         EdgeID &number_of_edge_based_nodes,
@@ -125,6 +126,19 @@ inline void readNodes(const boost::filesystem::path &path,
     util::serialization::read(reader, "/common/osm_node_ids", osm_node_ids);
 }
 
+// reads only coordinates from .osrm.nbg_nodes
+template <typename CoordinatesT>
+inline void readNodeCoordinates(const boost::filesystem::path &path,
+                      CoordinatesT &coordinates)
+{
+    static_assert(std::is_same<typename CoordinatesT::value_type, util::Coordinate>::value, "");
+
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    storage::serialization::read(reader, "/common/coordinates", coordinates);
+}
+
 // writes .osrm.nbg_nodes
 template <typename CoordinatesT, typename PackedOSMIDsT>
 inline void writeNodes(const boost::filesystem::path &path,
@@ -476,6 +490,24 @@ void readRamIndex(const boost::filesystem::path &path, RTreeT &rtree)
 
     util::serialization::read(reader, "/common/rtree", rtree);
 }
+
+template <typename EdgeListT>
+void writeCompressedNodeBasedGraph(const boost::filesystem::path &path, const EdgeListT &edge_list)
+{
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
+
+    storage::serialization::write(writer, "/extractor/cnbg", edge_list);
+}
+
+template <typename EdgeListT>
+void readCompressedNodeBasedGraph(const boost::filesystem::path &path, EdgeListT &edge_list)
+{
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    storage::serialization::read(reader, "/extractor/cnbg", edge_list);
+}
 }
 }
 }

include/partitioner/compressed_node_based_graph_reader.hpp

@@ -1,60 +0,0 @@
-#ifndef OSRM_PARTITIONER_COMPRESSED_NODE_BASED_GRAPH_READER_HPP
-#define OSRM_PARTITIONER_COMPRESSED_NODE_BASED_GRAPH_READER_HPP
-
-#include "storage/io.hpp"
-#include "util/coordinate.hpp"
-#include "util/typedefs.hpp"
-
-#include <string>
-#include <vector>
-
-namespace osrm
-{
-namespace partitioner
-{
-
-struct CompressedNodeBasedGraphEdge
-{
-    NodeID source;
-    NodeID target;
-};
-
-struct CompressedNodeBasedGraph
-{
-    CompressedNodeBasedGraph(storage::io::FileReader &reader)
-    {
-        // Reads:  | Fingerprint | #e | #n | edges | coordinates |
-        // - uint64: number of edges (from, to) pairs
-        // - uint64: number of nodes and therefore also coordinates
-        // - (uint32_t, uint32_t): num_edges * edges
-        // - (int32_t, int32_t: num_nodes * coordinates (lon, lat)
-        //
-        // Gets written in Extractor::WriteCompressedNodeBasedGraph
-
-        const auto num_edges = reader.ReadElementCount64();
-        const auto num_nodes = reader.ReadElementCount64();
-
-        edges.resize(num_edges);
-        coordinates.resize(num_nodes);
-
-        reader.ReadInto(edges);
-        reader.ReadInto(coordinates);
-    }
-
-    std::vector<CompressedNodeBasedGraphEdge> edges;
-    std::vector<util::Coordinate> coordinates;
-};
-
-inline CompressedNodeBasedGraph LoadCompressedNodeBasedGraph(const std::string &path)
-{
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader(path, fingerprint);
-
-    CompressedNodeBasedGraph graph{reader};
-    return graph;
-}
-
-} // ns partition
-} // ns osrm
-
-#endif

include/partitioner/partitioner_config.hpp

@@ -21,6 +21,7 @@ struct PartitionerConfig final : storage::IOConfig
                    {".osrm.ebg",
                     ".osrm.cnbg",
                     ".osrm.cnbg_to_ebg",
+                    ".osrm.nbg_nodes",
                     ".osrm.partition",
                     ".osrm.cells",
                     ".osrm.maneuver_overrides"}),

src/benchmarks/static_rtree.cpp

@@ -33,15 +33,6 @@ constexpr int32_t WORLD_MAX_LON = 180 * COORDINATE_PRECISION;
 using RTreeLeaf = extractor::EdgeBasedNodeSegment;
 using BenchStaticRTree = util::StaticRTree<RTreeLeaf, storage::Ownership::Container>;
 
-std::vector<util::Coordinate> loadCoordinates(const boost::filesystem::path &nodes_file)
-{
-    std::vector<util::Coordinate> coords;
-    extractor::PackedOSMIDs nodes;
-    extractor::files::readNodes(nodes_file, coords, nodes);
-
-    return coords;
-}
-
 template <typename QueryT>
 void benchmarkQuery(const std::vector<util::Coordinate> &queries,
                     const std::string &name,
@@ -99,7 +90,8 @@ int main(int argc, char **argv)
     const char *file_path = argv[2];
     const char *nodes_path = argv[3];
 
-    auto coords = osrm::benchmarks::loadCoordinates(nodes_path);
+    std::vector<osrm::util::Coordinate> coords;
+    osrm::extractor::files::readNodeCoordinates(nodes_path, coords);
 
     osrm::benchmarks::BenchStaticRTree rtree(file_path, coords);
     osrm::extractor::files::readRamIndex(ram_path, rtree);

src/extractor/extractor.cpp

@@ -14,6 +14,7 @@
 #include "extractor/restriction_parser.hpp"
 #include "extractor/scripting_environment.hpp"
 #include "extractor/name_table.hpp"
+#include "extractor/compressed_node_based_graph_edge.hpp"
 
 #include "guidance/files.hpp"
 #include "guidance/guidance_processing.hpp"
@@ -168,6 +169,26 @@ void SetExcludableClasses(const ExtractorCallbacks::ClassesMap &classes_map,
         }
     }
 }
+
+std::vector<CompressedNodeBasedGraphEdge> toEdgeList(const util::NodeBasedDynamicGraph& graph)
+{
+    std::vector<CompressedNodeBasedGraphEdge> edges;
+    edges.reserve(graph.GetNumberOfEdges());
+
+    // For all nodes iterate over its edges and dump (from, to) pairs
+    for (const NodeID from_node : util::irange(0u, graph.GetNumberOfNodes()))
+    {
+        for (const EdgeID edge : graph.GetAdjacentEdgeRange(from_node))
+        {
+            const auto to_node = graph.GetTarget(edge);
+
+            edges.push_back({from_node, to_node});
+        }
+    }
+
+    return edges;
+}
+
 }
 
 /**
@@ -267,10 +288,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
             compressed_node_based_graph_writing.wait();
     };
 
-    compressed_node_based_graph_writing = std::async(std::launch::async, [&] {
+    files::writeCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg").string(), toEdgeList(node_based_graph));
-        WriteCompressedNodeBasedGraph(
-            config.GetPath(".osrm.cnbg").string(), node_based_graph, coordinates);
-    });
 
     node_based_graph_factory.GetCompressedEdges().PrintStatistics();
 
@@ -814,48 +832,6 @@ void Extractor::BuildRTree(std::vector<EdgeBasedNodeSegment> edge_based_node_seg
     util::Log() << "finished r-tree construction in " << TIMER_SEC(construction) << " seconds";
 }
 
-void Extractor::WriteCompressedNodeBasedGraph(const std::string &path,
-                                              const util::NodeBasedDynamicGraph &graph,
-                                              const std::vector<util::Coordinate> &coordinates)
-{
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-
-    storage::io::FileWriter writer{path, fingerprint};
-
-    // Writes:  | Fingerprint | #e | #n | edges | coordinates |
-    // - uint64: number of edges (from, to) pairs
-    // - uint64: number of nodes and therefore also coordinates
-    // - (uint32_t, uint32_t): num_edges * edges
-    // - (int32_t, int32_t: num_nodes * coordinates (lon, lat)
-
-    const auto num_edges = graph.GetNumberOfEdges();
-    const auto num_nodes = graph.GetNumberOfNodes();
-
-    BOOST_ASSERT_MSG(num_nodes == coordinates.size(), "graph and embedding out of sync");
-
-    writer.WriteElementCount64(num_edges);
-    writer.WriteElementCount64(num_nodes);
-
-    // For all nodes iterate over its edges and dump (from, to) pairs
-    for (const NodeID from_node : util::irange(0u, num_nodes))
-    {
-        for (const EdgeID edge : graph.GetAdjacentEdgeRange(from_node))
-        {
-            const auto to_node = graph.GetTarget(edge);
-
-            writer.WriteFrom(from_node);
-            writer.WriteFrom(to_node);
-        }
-    }
-
-    // FIXME this is unneccesary: We have this data
-    for (const auto &qnode : coordinates)
-    {
-        writer.WriteFrom(qnode.lon);
-        writer.WriteFrom(qnode.lat);
-    }
-}
-
 template <typename Map> auto convertIDMapToVector(const Map &map)
 {
     std::vector<typename Map::key_type> result(map.size());

src/partitioner/partitioner.cpp

@@ -2,7 +2,6 @@
 #include "partitioner/bisection_graph.hpp"
 #include "partitioner/bisection_to_partition.hpp"
 #include "partitioner/cell_storage.hpp"
-#include "partitioner/compressed_node_based_graph_reader.hpp"
 #include "partitioner/edge_based_graph_reader.hpp"
 #include "partitioner/files.hpp"
 #include "partitioner/multi_level_partition.hpp"
@@ -10,6 +9,7 @@
 #include "partitioner/remove_unconnected.hpp"
 #include "partitioner/renumber.hpp"
 
+#include "extractor/compressed_node_based_graph_edge.hpp"
 #include "extractor/files.hpp"
 
 #include "util/coordinate.hpp"
@@ -40,19 +40,17 @@ namespace partitioner
 {
 auto getGraphBisection(const PartitionerConfig &config)
 {
-    auto compressed_node_based_graph =
+    std::vector<extractor::CompressedNodeBasedGraphEdge> edges;
-        LoadCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg").string());
+    extractor::files::readCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg"), edges);
+    groupEdgesBySource(begin(edges), end(edges));
 
-    util::Log() << "Loaded compressed node based graph: "
+    std::vector<util::Coordinate> coordinates;
-                << compressed_node_based_graph.edges.size() << " edges, "
+    extractor::files::readNodeCoordinates(config.GetPath(".osrm.nbg_nodes"), coordinates);
-                << compressed_node_based_graph.coordinates.size() << " nodes";
 
-    groupEdgesBySource(begin(compressed_node_based_graph.edges),
+    util::Log() << "Loaded compressed node based graph: " << edges.size() << " edges, "
-                       end(compressed_node_based_graph.edges));
+                << coordinates.size() << " nodes";
 
-    auto graph =
+    auto graph = makeBisectionGraph(coordinates, adaptToBisectionEdge(std::move(edges)));
-        makeBisectionGraph(compressed_node_based_graph.coordinates,
-                           adaptToBisectionEdge(std::move(compressed_node_based_graph.edges)));
 
     util::Log() << " running partition: " << config.max_cell_sizes.front() << " " << config.balance
                 << " " << config.boundary_factor << " " << config.num_optimizing_cuts << " "