Do not generate intermediate .osrm file in osrm-extract. (#6354)

include/extractor/extraction_containers.hpp

@@ -43,9 +43,6 @@ class ExtractionContainers
     void PrepareTrafficSignals(const ReferencedTrafficSignals &referenced_traffic_signals);
     void PrepareEdges(ScriptingEnvironment &scripting_environment);
 
-    void WriteNodes(storage::tar::FileWriter &file_out) const;
-    void WriteEdges(storage::tar::FileWriter &file_out) const;
-    void WriteMetadata(storage::tar::FileWriter &file_out) const;
     void WriteCharData(const std::string &file_name);
 
   public:
@@ -75,6 +72,8 @@ class ExtractionContainers
     std::vector<InputTrafficSignal> external_traffic_signals;
     TrafficSignals internal_traffic_signals;
 
+    std::vector<NodeBasedEdge> used_edges;
+
     // List of restrictions (conditional and unconditional) before we transform them into the
     // output types. Input containers reference OSMNodeIDs. We can only transform them to the
     // correct internal IDs after we've read everything. Without a multi-parse approach,
@@ -84,11 +83,12 @@ class ExtractionContainers
 
     std::vector<InputManeuverOverride> external_maneuver_overrides_list;
     std::vector<UnresolvedManeuverOverride> internal_maneuver_overrides;
+    std::unordered_set<NodeID> used_barrier_nodes;
+    NodeVector used_nodes;
 
     ExtractionContainers();
 
     void PrepareData(ScriptingEnvironment &scripting_environment,
-                     const std::string &osrm_path,
                      const std::string &names_data_path);
 };
 } // namespace extractor

include/extractor/extractor.hpp

@@ -60,14 +60,25 @@ class Extractor
     Extractor(ExtractorConfig extractor_config) : config(std::move(extractor_config)) {}
     int run(ScriptingEnvironment &scripting_environment);
 
+  private:
+    struct ParsedOSMData
+    {
+        LaneDescriptionMap turn_lane_map;
+        std::vector<TurnRestriction> turn_restrictions;
+        std::vector<UnresolvedManeuverOverride> unresolved_maneuver_overrides;
+        TrafficSignals traffic_signals;
+        std::unordered_set<NodeID> barriers;
+        std::vector<util::Coordinate> osm_coordinates;
+        extractor::PackedOSMIDs osm_node_ids;
+        std::vector<NodeBasedEdge> edge_list;
+        std::vector<NodeBasedEdgeAnnotation> annotation_data;
+    };
+
   private:
     ExtractorConfig config;
 
-    std::tuple<LaneDescriptionMap,
-               std::vector<TurnRestriction>,
-               std::vector<UnresolvedManeuverOverride>,
-               TrafficSignals>
-    ParseOSMData(ScriptingEnvironment &scripting_environment, const unsigned number_of_threads);
+    ParsedOSMData ParseOSMData(ScriptingEnvironment &scripting_environment,
+                               const unsigned number_of_threads);
 
     EdgeID BuildEdgeExpandedGraph(
         // input data

include/extractor/extractor_config.hpp

@@ -48,7 +48,7 @@ struct ExtractorConfig final : storage::IOConfig
                   "",
               },
               {},
-              {".osrm",
+              {".osrm.nbg",
                ".osrm.restrictions",
                ".osrm.names",
                ".osrm.tls",
@@ -89,6 +89,7 @@ struct ExtractorConfig final : storage::IOConfig
     bool use_metadata = false;
     bool parse_conditionals = false;
     bool use_locations_cache = true;
+    bool dump_nbg_graph = false;
 };
 } // namespace extractor
 } // namespace osrm

include/extractor/files.hpp

@@ -444,13 +444,11 @@ inline void readConditionalRestrictions(const boost::filesystem::path &path,
 }
 
 // reads .osrm file which is a temporary file of osrm-extract
-template <typename BarrierOutIter, typename PackedOSMIDsT>
+template <typename PackedOSMIDsT>
 void readRawNBGraph(const boost::filesystem::path &path,
-                    BarrierOutIter barriers,
                     std::vector<util::Coordinate> &coordinates,
                     PackedOSMIDsT &osm_node_ids,
-                    std::vector<extractor::NodeBasedEdge> &edge_list,
-                    std::vector<extractor::NodeBasedEdgeAnnotation> &annotations)
+                    std::vector<extractor::NodeBasedEdge> &edge_list)
 {
     const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
     storage::tar::FileReader reader{path, fingerprint};
@@ -468,10 +466,7 @@ void readRawNBGraph(const boost::filesystem::path &path,
     reader.ReadStreaming<extractor::QueryNode>("/extractor/nodes",
                                                boost::make_function_output_iterator(decode));
 
-    reader.ReadStreaming<NodeID>("/extractor/barriers", barriers);
-
     storage::serialization::read(reader, "/extractor/edges", edge_list);
-    storage::serialization::read(reader, "/extractor/annotations", annotations);
 }
 
 template <typename NameTableT>

include/extractor/node_based_graph_factory.hpp

@@ -34,15 +34,19 @@ namespace extractor
 class NodeBasedGraphFactory
 {
   public:
-    // The node-based graph factory loads the *.osrm file and transforms the data within into the
+    // The node-based graph factory transforms the graph data into the
     // node-based graph to represent the OSM network. This includes geometry compression, annotation
     // data optimisation and many other aspects. After this step, the edge-based graph factory can
     // turn the graph into the routing graph to be used with the navigation algorithms.
-    NodeBasedGraphFactory(const boost::filesystem::path &input_file,
-                          ScriptingEnvironment &scripting_environment,
+    NodeBasedGraphFactory(ScriptingEnvironment &scripting_environment,
                           std::vector<TurnRestriction> &turn_restrictions,
                           std::vector<UnresolvedManeuverOverride> &maneuver_overrides,
-                          const TrafficSignals &traffic_signals);
+                          const TrafficSignals &traffic_signals,
+                          std::unordered_set<NodeID> &&barriers,
+                          std::vector<util::Coordinate> &&coordinates,
+                          extractor::PackedOSMIDs &&osm_node_ids,
+                          const std::vector<NodeBasedEdge> &edge_list,
+                          std::vector<NodeBasedEdgeAnnotation> &&annotation_data);
 
     auto const &GetGraph() const { return compressed_output_graph; }
     auto const &GetBarriers() const { return barriers; }
@@ -60,9 +64,8 @@ class NodeBasedGraphFactory
     void ReleaseOsmNodes();
 
   private:
-    // Get the information from the *.osrm file (direct product of the extractor callback/extraction
-    // containers) and prepare the graph creation process
-    void LoadDataFromFile(const boost::filesystem::path &input_file);
+    // Build and validate compressed output graph
+    void BuildCompressedOutputGraph(const std::vector<NodeBasedEdge> &edge_list);
 
     // Compress the node-based graph into a compact representation of itself. This removes storing a
     // single edge for every part of the geometry and might also combine meta-data for multiple

include/partitioner/partitioner_config.hpp

@@ -16,7 +16,7 @@ namespace partitioner
 struct PartitionerConfig final : storage::IOConfig
 {
     PartitionerConfig()
-        : IOConfig({".osrm", ".osrm.fileIndex", ".osrm.ebg_nodes", ".osrm.enw"},
+        : IOConfig({".osrm.fileIndex", ".osrm.ebg_nodes", ".osrm.enw"},
                    {".osrm.hsgr", ".osrm.cnbg"},
                    {".osrm.ebg",
                     ".osrm.cnbg",