Port .cnbg file to tar format

src/benchmarks/static_rtree.cpp

@@ -33,15 +33,6 @@ constexpr int32_t WORLD_MAX_LON = 180 * COORDINATE_PRECISION;
 using RTreeLeaf = extractor::EdgeBasedNodeSegment;
 using BenchStaticRTree = util::StaticRTree<RTreeLeaf, storage::Ownership::Container>;
 
-std::vector<util::Coordinate> loadCoordinates(const boost::filesystem::path &nodes_file)
-{
-    std::vector<util::Coordinate> coords;
-    extractor::PackedOSMIDs nodes;
-    extractor::files::readNodes(nodes_file, coords, nodes);
-
-    return coords;
-}
-
 template <typename QueryT>
 void benchmarkQuery(const std::vector<util::Coordinate> &queries,
                     const std::string &name,
@@ -99,7 +90,8 @@ int main(int argc, char **argv)
     const char *file_path = argv[2];
     const char *nodes_path = argv[3];
 
-    auto coords = osrm::benchmarks::loadCoordinates(nodes_path);
+    std::vector<osrm::util::Coordinate> coords;
+    osrm::extractor::files::readNodeCoordinates(nodes_path, coords);
 
     osrm::benchmarks::BenchStaticRTree rtree(file_path, coords);
     osrm::extractor::files::readRamIndex(ram_path, rtree);

src/extractor/extractor.cpp

@@ -14,6 +14,7 @@
 #include "extractor/restriction_parser.hpp"
 #include "extractor/scripting_environment.hpp"
 #include "extractor/name_table.hpp"
+#include "extractor/compressed_node_based_graph_edge.hpp"
 
 #include "guidance/files.hpp"
 #include "guidance/guidance_processing.hpp"
@@ -168,6 +169,26 @@ void SetExcludableClasses(const ExtractorCallbacks::ClassesMap &classes_map,
         }
     }
 }
+
+std::vector<CompressedNodeBasedGraphEdge> toEdgeList(const util::NodeBasedDynamicGraph& graph)
+{
+    std::vector<CompressedNodeBasedGraphEdge> edges;
+    edges.reserve(graph.GetNumberOfEdges());
+
+    // For all nodes iterate over its edges and dump (from, to) pairs
+    for (const NodeID from_node : util::irange(0u, graph.GetNumberOfNodes()))
+    {
+        for (const EdgeID edge : graph.GetAdjacentEdgeRange(from_node))
+        {
+            const auto to_node = graph.GetTarget(edge);
+
+            edges.push_back({from_node, to_node});
+        }
+    }
+
+    return edges;
+}
+
 }
 
 /**
@@ -267,10 +288,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
             compressed_node_based_graph_writing.wait();
     };
 
-    compressed_node_based_graph_writing = std::async(std::launch::async, [&] {
-        WriteCompressedNodeBasedGraph(
-            config.GetPath(".osrm.cnbg").string(), node_based_graph, coordinates);
-    });
+    files::writeCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg").string(), toEdgeList(node_based_graph));
 
     node_based_graph_factory.GetCompressedEdges().PrintStatistics();
 
@@ -814,48 +832,6 @@ void Extractor::BuildRTree(std::vector<EdgeBasedNodeSegment> edge_based_node_seg
     util::Log() << "finished r-tree construction in " << TIMER_SEC(construction) << " seconds";
 }
 
-void Extractor::WriteCompressedNodeBasedGraph(const std::string &path,
-                                              const util::NodeBasedDynamicGraph &graph,
-                                              const std::vector<util::Coordinate> &coordinates)
-{
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-
-    storage::io::FileWriter writer{path, fingerprint};
-
-    // Writes:  | Fingerprint | #e | #n | edges | coordinates |
-    // - uint64: number of edges (from, to) pairs
-    // - uint64: number of nodes and therefore also coordinates
-    // - (uint32_t, uint32_t): num_edges * edges
-    // - (int32_t, int32_t: num_nodes * coordinates (lon, lat)
-
-    const auto num_edges = graph.GetNumberOfEdges();
-    const auto num_nodes = graph.GetNumberOfNodes();
-
-    BOOST_ASSERT_MSG(num_nodes == coordinates.size(), "graph and embedding out of sync");
-
-    writer.WriteElementCount64(num_edges);
-    writer.WriteElementCount64(num_nodes);
-
-    // For all nodes iterate over its edges and dump (from, to) pairs
-    for (const NodeID from_node : util::irange(0u, num_nodes))
-    {
-        for (const EdgeID edge : graph.GetAdjacentEdgeRange(from_node))
-        {
-            const auto to_node = graph.GetTarget(edge);
-
-            writer.WriteFrom(from_node);
-            writer.WriteFrom(to_node);
-        }
-    }
-
-    // FIXME this is unneccesary: We have this data
-    for (const auto &qnode : coordinates)
-    {
-        writer.WriteFrom(qnode.lon);
-        writer.WriteFrom(qnode.lat);
-    }
-}
-
 template <typename Map> auto convertIDMapToVector(const Map &map)
 {
     std::vector<typename Map::key_type> result(map.size());

src/partitioner/partitioner.cpp

@@ -2,7 +2,6 @@
 #include "partitioner/bisection_graph.hpp"
 #include "partitioner/bisection_to_partition.hpp"
 #include "partitioner/cell_storage.hpp"
-#include "partitioner/compressed_node_based_graph_reader.hpp"
 #include "partitioner/edge_based_graph_reader.hpp"
 #include "partitioner/files.hpp"
 #include "partitioner/multi_level_partition.hpp"
@@ -10,6 +9,7 @@
 #include "partitioner/remove_unconnected.hpp"
 #include "partitioner/renumber.hpp"
 
+#include "extractor/compressed_node_based_graph_edge.hpp"
 #include "extractor/files.hpp"
 
 #include "util/coordinate.hpp"
@@ -40,19 +40,17 @@ namespace partitioner
 {
 auto getGraphBisection(const PartitionerConfig &config)
 {
-    auto compressed_node_based_graph =
-        LoadCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg").string());
+    std::vector<extractor::CompressedNodeBasedGraphEdge> edges;
+    extractor::files::readCompressedNodeBasedGraph(config.GetPath(".osrm.cnbg"), edges);
+    groupEdgesBySource(begin(edges), end(edges));
 
-    util::Log() << "Loaded compressed node based graph: "
-                << compressed_node_based_graph.edges.size() << " edges, "
-                << compressed_node_based_graph.coordinates.size() << " nodes";
+    std::vector<util::Coordinate> coordinates;
+    extractor::files::readNodeCoordinates(config.GetPath(".osrm.nbg_nodes"), coordinates);
 
-    groupEdgesBySource(begin(compressed_node_based_graph.edges),
-                       end(compressed_node_based_graph.edges));
+    util::Log() << "Loaded compressed node based graph: " << edges.size() << " edges, "
+                << coordinates.size() << " nodes";
 
-    auto graph =
-        makeBisectionGraph(compressed_node_based_graph.coordinates,
-                           adaptToBisectionEdge(std::move(compressed_node_based_graph.edges)));
+    auto graph = makeBisectionGraph(coordinates, adaptToBisectionEdge(std::move(edges)));
 
     util::Log() << " running partition: " << config.max_cell_sizes.front() << " " << config.balance
                 << " " << config.boundary_factor << " " << config.num_optimizing_cuts << " "