Do not generate intermediate .osrm file in osrm-extract. (#6354)

.github/workflows/osrm-backend.yml

@@ -124,7 +124,17 @@ jobs:
           # when `--memory-swap` value equals `--memory` it means container won't use swap
           # see https://docs.docker.com/config/containers/resource_constraints/#--memory-swap-details
           MEMORY_ARGS="--memory=1g --memory-swap=1g"
-          docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-extract -p /opt/car.lua /data/berlin-latest.osm.pbf
+          docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-extract --dump-nbg-graph -p /opt/car.lua /data/berlin-latest.osm.pbf
+          docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-components /data/berlin-latest.osrm.nbg /data/berlin-latest.geojson
+          if [ ! -s "${PWD}/berlin-latest.geojson" ]
+          then
+            >&2 echo "No berlin-latest.geojson found"
+            exit 1
+          fi
+
+          # removing `.osrm.nbg` to check that whole pipeline works without it
+          rm -rf "${PWD}/berlin-latest.osrm.nbg"
+
           docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-partition /data/berlin-latest.osrm
           docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-customize /data/berlin-latest.osrm
           docker run $MEMORY_ARGS --name=osrm-container -t -p 5000:5000 -v "${PWD}:/data" "${TAG}" osrm-routed --algorithm mld /data/berlin-latest.osrm &

CHANGELOG.md

@@ -64,7 +64,8 @@
       - FIXED: Ensure u-turn exists in intersection view. [#6376](https://github.com/Project-OSRM/osrm-backend/pull/6376)
     - Profile:
       - CHANGED: Bicycle surface speeds [#6212](https://github.com/Project-OSRM/osrm-backend/pull/6212)
-
+    - Tools:
+      - CHANGED: Do not generate intermediate .osrm file in osrm-extract. [#6354](https://github.com/Project-OSRM/osrm-backend/pull/6354)
 
 # 5.26.0
   - Changes from 5.25.0

README.md

@@ -65,7 +65,7 @@ The flag `-v "${PWD}:/data"` creates the directory `/data` inside the docker con
     docker run -t -v "${PWD}:/data" osrm/osrm-backend osrm-partition /data/berlin-latest.osrm
     docker run -t -v "${PWD}:/data" osrm/osrm-backend osrm-customize /data/berlin-latest.osrm
 
-Note that `berlin-latest.osrm` has a different file extension. 
+Note there is no `berlin-latest.osrm` file, but multiple `berlin-latest.osrm.*` files, i.e. `berlin-latest.osrm` is not file path, but "base" path referring to set of files and there is an option to omit this `.osrm` suffix completely(e.g. `osrm-partition /data/berlin-latest`). 
 
     docker run -t -i -p 5000:5000 -v "${PWD}:/data" osrm/osrm-backend osrm-routed --algorithm mld /data/berlin-latest.osrm
 

docs/nodejs/api.md

@@ -3,8 +3,8 @@
 ## OSRM
 
 The `OSRM` method is the main constructor for creating an OSRM instance.
-An OSRM instance requires a `.osrm` dataset, which is prepared by the OSRM toolchain.
+An OSRM instance requires a `.osrm.*` dataset(`.osrm.*` because it contains several files), which is prepared by the OSRM toolchain.
-You can create such a `.osrm` file by running the OSRM binaries we ship in `node_modules/osrm/lib/binding/` and default
+You can create such a `.osrm.*` dataset by running the OSRM binaries we ship in `node_modules/osrm/lib/binding/` and default
 profiles (e.g. for setting speeds and determining road types to route on) in `node_modules/osrm/profiles/`:
 
     node_modules/osrm/lib/binding/osrm-extract data.osm.pbf -p node_modules/osrm/profiles/car.lua
@@ -12,7 +12,7 @@ profiles (e.g. for setting speeds and determining road types to route on) in `no
 
 Consult the [osrm-backend](https://github.com/Project-OSRM/osrm-backend) documentation for further details.
 
-Once you have a complete `network.osrm` file, you can calculate routes in javascript with this object.
+Once you have a complete `network.osrm.*` dataset, you can calculate routes in javascript with this object.
 
 ```javascript
 var osrm = new OSRM('network.osrm');

include/extractor/extraction_containers.hpp

@@ -43,9 +43,6 @@ class ExtractionContainers
     void PrepareTrafficSignals(const ReferencedTrafficSignals &referenced_traffic_signals);
     void PrepareEdges(ScriptingEnvironment &scripting_environment);
 
-    void WriteNodes(storage::tar::FileWriter &file_out) const;
-    void WriteEdges(storage::tar::FileWriter &file_out) const;
-    void WriteMetadata(storage::tar::FileWriter &file_out) const;
     void WriteCharData(const std::string &file_name);
 
   public:
@@ -75,6 +72,8 @@ class ExtractionContainers
     std::vector<InputTrafficSignal> external_traffic_signals;
     TrafficSignals internal_traffic_signals;
 
+    std::vector<NodeBasedEdge> used_edges;
+
     // List of restrictions (conditional and unconditional) before we transform them into the
     // output types. Input containers reference OSMNodeIDs. We can only transform them to the
     // correct internal IDs after we've read everything. Without a multi-parse approach,
@@ -84,11 +83,12 @@ class ExtractionContainers
 
     std::vector<InputManeuverOverride> external_maneuver_overrides_list;
     std::vector<UnresolvedManeuverOverride> internal_maneuver_overrides;
+    std::unordered_set<NodeID> used_barrier_nodes;
+    NodeVector used_nodes;
 
     ExtractionContainers();
 
     void PrepareData(ScriptingEnvironment &scripting_environment,
-                     const std::string &osrm_path,
                      const std::string &names_data_path);
 };
 } // namespace extractor

include/extractor/extractor.hpp

@@ -60,14 +60,25 @@ class Extractor
     Extractor(ExtractorConfig extractor_config) : config(std::move(extractor_config)) {}
     int run(ScriptingEnvironment &scripting_environment);
 
+  private:
+    struct ParsedOSMData
+    {
+        LaneDescriptionMap turn_lane_map;
+        std::vector<TurnRestriction> turn_restrictions;
+        std::vector<UnresolvedManeuverOverride> unresolved_maneuver_overrides;
+        TrafficSignals traffic_signals;
+        std::unordered_set<NodeID> barriers;
+        std::vector<util::Coordinate> osm_coordinates;
+        extractor::PackedOSMIDs osm_node_ids;
+        std::vector<NodeBasedEdge> edge_list;
+        std::vector<NodeBasedEdgeAnnotation> annotation_data;
+    };
+
   private:
     ExtractorConfig config;
 
-    std::tuple<LaneDescriptionMap,
+    ParsedOSMData ParseOSMData(ScriptingEnvironment &scripting_environment,
-               std::vector<TurnRestriction>,
+                               const unsigned number_of_threads);
-               std::vector<UnresolvedManeuverOverride>,
-               TrafficSignals>
-    ParseOSMData(ScriptingEnvironment &scripting_environment, const unsigned number_of_threads);
 
     EdgeID BuildEdgeExpandedGraph(
         // input data

include/extractor/extractor_config.hpp

@@ -48,7 +48,7 @@ struct ExtractorConfig final : storage::IOConfig
                   "",
               },
               {},
-              {".osrm",
+              {".osrm.nbg",
                ".osrm.restrictions",
                ".osrm.names",
                ".osrm.tls",
@@ -89,6 +89,7 @@ struct ExtractorConfig final : storage::IOConfig
     bool use_metadata = false;
     bool parse_conditionals = false;
     bool use_locations_cache = true;
+    bool dump_nbg_graph = false;
 };
 } // namespace extractor
 } // namespace osrm

include/extractor/files.hpp

@@ -444,13 +444,11 @@ inline void readConditionalRestrictions(const boost::filesystem::path &path,
 }
 
 // reads .osrm file which is a temporary file of osrm-extract
-template <typename BarrierOutIter, typename PackedOSMIDsT>
+template <typename PackedOSMIDsT>
 void readRawNBGraph(const boost::filesystem::path &path,
-                    BarrierOutIter barriers,
                     std::vector<util::Coordinate> &coordinates,
                     PackedOSMIDsT &osm_node_ids,
-                    std::vector<extractor::NodeBasedEdge> &edge_list,
+                    std::vector<extractor::NodeBasedEdge> &edge_list)
-                    std::vector<extractor::NodeBasedEdgeAnnotation> &annotations)
 {
     const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
     storage::tar::FileReader reader{path, fingerprint};
@@ -468,10 +466,7 @@ void readRawNBGraph(const boost::filesystem::path &path,
     reader.ReadStreaming<extractor::QueryNode>("/extractor/nodes",
                                                boost::make_function_output_iterator(decode));
 
-    reader.ReadStreaming<NodeID>("/extractor/barriers", barriers);
-
     storage::serialization::read(reader, "/extractor/edges", edge_list);
-    storage::serialization::read(reader, "/extractor/annotations", annotations);
 }
 
 template <typename NameTableT>

include/extractor/node_based_graph_factory.hpp

@@ -34,15 +34,19 @@ namespace extractor
 class NodeBasedGraphFactory
 {
   public:
-    // The node-based graph factory loads the *.osrm file and transforms the data within into the
+    // The node-based graph factory transforms the graph data into the
     // node-based graph to represent the OSM network. This includes geometry compression, annotation
     // data optimisation and many other aspects. After this step, the edge-based graph factory can
     // turn the graph into the routing graph to be used with the navigation algorithms.
-    NodeBasedGraphFactory(const boost::filesystem::path &input_file,
+    NodeBasedGraphFactory(ScriptingEnvironment &scripting_environment,
-                          ScriptingEnvironment &scripting_environment,
                           std::vector<TurnRestriction> &turn_restrictions,
                           std::vector<UnresolvedManeuverOverride> &maneuver_overrides,
-                          const TrafficSignals &traffic_signals);
+                          const TrafficSignals &traffic_signals,
+                          std::unordered_set<NodeID> &&barriers,
+                          std::vector<util::Coordinate> &&coordinates,
+                          extractor::PackedOSMIDs &&osm_node_ids,
+                          const std::vector<NodeBasedEdge> &edge_list,
+                          std::vector<NodeBasedEdgeAnnotation> &&annotation_data);
 
     auto const &GetGraph() const { return compressed_output_graph; }
     auto const &GetBarriers() const { return barriers; }
@@ -60,9 +64,8 @@ class NodeBasedGraphFactory
     void ReleaseOsmNodes();
 
   private:
-    // Get the information from the *.osrm file (direct product of the extractor callback/extraction
+    // Build and validate compressed output graph
-    // containers) and prepare the graph creation process
+    void BuildCompressedOutputGraph(const std::vector<NodeBasedEdge> &edge_list);
-    void LoadDataFromFile(const boost::filesystem::path &input_file);
 
     // Compress the node-based graph into a compact representation of itself. This removes storing a
     // single edge for every part of the geometry and might also combine meta-data for multiple

include/partitioner/partitioner_config.hpp

@@ -16,7 +16,7 @@ namespace partitioner
 struct PartitionerConfig final : storage::IOConfig
 {
     PartitionerConfig()
-        : IOConfig({".osrm", ".osrm.fileIndex", ".osrm.ebg_nodes", ".osrm.enw"},
+        : IOConfig({".osrm.fileIndex", ".osrm.ebg_nodes", ".osrm.enw"},
                    {".osrm.hsgr", ".osrm.cnbg"},
                    {".osrm.ebg",
                     ".osrm.cnbg",

src/extractor/extraction_containers.cpp

@@ -5,8 +5,8 @@
 #include "extractor/name_table.hpp"
 #include "extractor/restriction.hpp"
 #include "extractor/serialization.hpp"
-
 #include "util/coordinate_calculation.hpp"
+#include "util/integer_range.hpp"
 
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
@@ -16,6 +16,7 @@
 #include "util/timing_util.hpp"
 
 #include <boost/assert.hpp>
+#include <boost/core/ignore_unused.hpp>
 #include <boost/numeric/conversion/cast.hpp>
 
 #include <tbb/parallel_sort.h>
@@ -407,22 +408,14 @@ ExtractionContainers::ExtractionContainers()
  *
  */
 void ExtractionContainers::PrepareData(ScriptingEnvironment &scripting_environment,
-                                       const std::string &osrm_path,
                                        const std::string &name_file_name)
 {
-    storage::tar::FileWriter writer(osrm_path, storage::tar::FileWriter::GenerateFingerprint);
-
     const auto restriction_ways = IdentifyRestrictionWays();
     const auto maneuver_override_ways = IdentifyManeuverOverrideWays();
     const auto traffic_signals = IdentifyTrafficSignals();
 
     PrepareNodes();
-    WriteNodes(writer);
     PrepareEdges(scripting_environment);
-    all_nodes_list.clear(); // free all_nodes_list before allocation of normal_edges
-    all_nodes_list.shrink_to_fit();
-    WriteEdges(writer);
-    WriteMetadata(writer);
 
     PrepareTrafficSignals(traffic_signals);
     PrepareManeuverOverrides(maneuver_override_ways);
@@ -519,6 +512,60 @@ void ExtractionContainers::PrepareNodes()
         TIMER_STOP(id_map);
         log << "ok, after " << TIMER_SEC(id_map) << "s";
     }
+    {
+        util::UnbufferedLog log;
+        log << "Confirming/Writing used nodes     ... ";
+        TIMER_START(write_nodes);
+        // identify all used nodes by a merging step of two sorted lists
+        auto node_iterator = all_nodes_list.begin();
+        auto node_id_iterator = used_node_id_list.begin();
+        const auto all_nodes_list_end = all_nodes_list.end();
+
+        for (const auto index : util::irange<NodeID>(0, used_node_id_list.size()))
+        {
+            boost::ignore_unused(index);
+            BOOST_ASSERT(node_id_iterator != used_node_id_list.end());
+            BOOST_ASSERT(node_iterator != all_nodes_list_end);
+            BOOST_ASSERT(*node_id_iterator >= node_iterator->node_id);
+            while (*node_id_iterator > node_iterator->node_id &&
+                   node_iterator != all_nodes_list_end)
+            {
+                ++node_iterator;
+            }
+            if (node_iterator == all_nodes_list_end || *node_id_iterator < node_iterator->node_id)
+            {
+                throw util::exception(
+                    "Invalid OSM data: Referenced non-existing node with ID " +
+                    std::to_string(static_cast<std::uint64_t>(*node_id_iterator)));
+            }
+            BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
+
+            ++node_id_iterator;
+
+            used_nodes.emplace_back(*node_iterator++);
+        }
+
+        TIMER_STOP(write_nodes);
+        log << "ok, after " << TIMER_SEC(write_nodes) << "s";
+    }
+
+    {
+        util::UnbufferedLog log;
+        log << "Writing barrier nodes     ... ";
+        TIMER_START(write_nodes);
+        for (const auto osm_id : barrier_nodes)
+        {
+            const auto node_id = mapExternalToInternalNodeID(
+                used_node_id_list.begin(), used_node_id_list.end(), osm_id);
+            if (node_id != SPECIAL_NODEID)
+            {
+                used_barrier_nodes.emplace(node_id);
+            }
+        }
+        log << "ok, after " << TIMER_SEC(write_nodes) << "s";
+    }
+
+    util::Log() << "Processed " << max_internal_node_id << " nodes";
 }
 
 void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environment)
@@ -804,12 +851,11 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
             all_edges_list[j].result.target = SPECIAL_NODEID;
         }
     }
-}
 
-void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) const
+    all_nodes_list.clear(); // free all_nodes_list before allocation of used_edges
-{
+    all_nodes_list.shrink_to_fit();
-    std::vector<NodeBasedEdge> normal_edges;
+
-    normal_edges.reserve(all_edges_list.size());
+    used_edges.reserve(all_edges_list.size());
     {
         util::UnbufferedLog log;
         log << "Writing used edges       ... " << std::flush;
@@ -825,98 +871,20 @@ void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) const
 
             // IMPORTANT: here, we're using slicing to only write the data from the base
             // class of NodeBasedEdgeWithOSM
-            normal_edges.push_back(edge.result);
+            used_edges.push_back(edge.result);
         }
 
-        if (normal_edges.size() > std::numeric_limits<uint32_t>::max())
+        if (used_edges.size() > std::numeric_limits<uint32_t>::max())
         {
             throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF);
         }
 
-        storage::serialization::write(writer, "/extractor/edges", normal_edges);
-
         TIMER_STOP(write_edges);
         log << "ok, after " << TIMER_SEC(write_edges) << "s";
-        log << " -- Processed " << normal_edges.size() << " edges";
+        log << " -- Processed " << used_edges.size() << " edges";
     }
 }
 
-void ExtractionContainers::WriteMetadata(storage::tar::FileWriter &writer) const
-{
-    util::UnbufferedLog log;
-    log << "Writing way meta-data     ... " << std::flush;
-    TIMER_START(write_meta_data);
-
-    storage::serialization::write(writer, "/extractor/annotations", all_edges_annotation_data_list);
-
-    TIMER_STOP(write_meta_data);
-    log << "ok, after " << TIMER_SEC(write_meta_data) << "s";
-    log << " -- Metadata contains << " << all_edges_annotation_data_list.size() << " entries.";
-}
-
-void ExtractionContainers::WriteNodes(storage::tar::FileWriter &writer) const
-{
-    {
-        util::UnbufferedLog log;
-        log << "Confirming/Writing used nodes     ... ";
-        TIMER_START(write_nodes);
-        // identify all used nodes by a merging step of two sorted lists
-        auto node_iterator = all_nodes_list.begin();
-        auto node_id_iterator = used_node_id_list.begin();
-        const auto all_nodes_list_end = all_nodes_list.end();
-
-        const std::function<QueryNode()> encode_function = [&]() -> QueryNode {
-            BOOST_ASSERT(node_id_iterator != used_node_id_list.end());
-            BOOST_ASSERT(node_iterator != all_nodes_list_end);
-            BOOST_ASSERT(*node_id_iterator >= node_iterator->node_id);
-            while (*node_id_iterator > node_iterator->node_id &&
-                   node_iterator != all_nodes_list_end)
-            {
-                ++node_iterator;
-            }
-            if (node_iterator == all_nodes_list_end || *node_id_iterator < node_iterator->node_id)
-            {
-                throw util::exception(
-                    "Invalid OSM data: Referenced non-existing node with ID " +
-                    std::to_string(static_cast<std::uint64_t>(*node_id_iterator)));
-            }
-            BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
-
-            ++node_id_iterator;
-            return *node_iterator++;
-        };
-
-        writer.WriteElementCount64("/extractor/nodes", used_node_id_list.size());
-        writer.WriteStreaming<QueryNode>(
-            "/extractor/nodes",
-            boost::make_function_input_iterator(encode_function, boost::infinite()),
-            used_node_id_list.size());
-
-        TIMER_STOP(write_nodes);
-        log << "ok, after " << TIMER_SEC(write_nodes) << "s";
-    }
-
-    {
-        util::UnbufferedLog log;
-        log << "Writing barrier nodes     ... ";
-        TIMER_START(write_nodes);
-        std::vector<NodeID> internal_barrier_nodes;
-        for (const auto osm_id : barrier_nodes)
-        {
-            const auto node_id = mapExternalToInternalNodeID(
-                used_node_id_list.begin(), used_node_id_list.end(), osm_id);
-            if (node_id != SPECIAL_NODEID)
-            {
-                internal_barrier_nodes.push_back(node_id);
-            }
-        }
-        storage::serialization::write(writer, "/extractor/barriers", internal_barrier_nodes);
-        log << "ok, after " << TIMER_SEC(write_nodes) << "s";
-    }
-
-    util::Log() << "Processed " << max_internal_node_id << " nodes";
-}
-
 ExtractionContainers::ReferencedWays ExtractionContainers::IdentifyManeuverOverrideWays()
 {
     ReferencedWays maneuver_override_ways;

src/extractor/extractor.cpp

@@ -204,12 +204,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
     tbb::global_control gc(tbb::global_control::max_allowed_parallelism,
                            config.requested_num_threads);
 
-    LaneDescriptionMap turn_lane_map;
+    auto parsed_osm_data = ParseOSMData(scripting_environment, number_of_threads);
-    std::vector<TurnRestriction> turn_restrictions;
-    std::vector<UnresolvedManeuverOverride> unresolved_maneuver_overrides;
-    TrafficSignals traffic_signals;
-    std::tie(turn_lane_map, turn_restrictions, unresolved_maneuver_overrides, traffic_signals) =
-        ParseOSMData(scripting_environment, number_of_threads);
 
     // Transform the node-based graph that OSM is based on into an edge-based graph
     // that is better for routing.  Every edge becomes a node, and every valid
@@ -227,11 +222,15 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
     std::uint32_t ebg_connectivity_checksum = 0;
 
     // Create a node-based graph from the OSRM file
-    NodeBasedGraphFactory node_based_graph_factory(config.GetPath(".osrm"),
+    NodeBasedGraphFactory node_based_graph_factory(scripting_environment,
-                                                   scripting_environment,
+                                                   parsed_osm_data.turn_restrictions,
-                                                   turn_restrictions,
+                                                   parsed_osm_data.unresolved_maneuver_overrides,
-                                                   unresolved_maneuver_overrides,
+                                                   parsed_osm_data.traffic_signals,
-                                                   traffic_signals);
+                                                   std::move(parsed_osm_data.barriers),
+                                                   std::move(parsed_osm_data.osm_coordinates),
+                                                   std::move(parsed_osm_data.osm_node_ids),
+                                                   parsed_osm_data.edge_list,
+                                                   std::move(parsed_osm_data.annotation_data));
 
     NameTable name_table;
     files::readNames(config.GetPath(".osrm.names"), name_table);
@@ -270,10 +269,11 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
     edge_based_nodes_container =
         EdgeBasedNodeDataContainer({}, std::move(node_based_graph_factory.GetAnnotationData()));
 
-    turn_restrictions = removeInvalidTurnPaths(std::move(turn_restrictions), node_based_graph);
+    parsed_osm_data.turn_restrictions =
-    unresolved_maneuver_overrides =
+        removeInvalidTurnPaths(std::move(parsed_osm_data.turn_restrictions), node_based_graph);
-        removeInvalidTurnPaths(std::move(unresolved_maneuver_overrides), node_based_graph);
+    parsed_osm_data.unresolved_maneuver_overrides = removeInvalidTurnPaths(
-    auto restriction_graph = constructRestrictionGraph(turn_restrictions);
+        std::move(parsed_osm_data.unresolved_maneuver_overrides), node_based_graph);
+    auto restriction_graph = constructRestrictionGraph(parsed_osm_data.turn_restrictions);
 
     const auto number_of_node_based_nodes = node_based_graph.GetNumberOfNodes();
 
@@ -282,12 +282,12 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
                                coordinates,
                                node_based_graph_factory.GetCompressedEdges(),
                                barrier_nodes,
-                               traffic_signals,
+                               parsed_osm_data.traffic_signals,
                                restriction_graph,
                                segregated_edges,
                                name_table,
-                               unresolved_maneuver_overrides,
+                               parsed_osm_data.unresolved_maneuver_overrides,
-                               turn_lane_map,
+                               parsed_osm_data.turn_lane_map,
                                scripting_environment,
                                edge_based_nodes_container,
                                edge_based_node_segments,
@@ -304,7 +304,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
                          barrier_nodes,
                          restriction_graph,
                          name_table,
-                         std::move(turn_lane_map),
+                         std::move(parsed_osm_data.turn_lane_map),
                          scripting_environment);
 
     TIMER_STOP(expansion);
@@ -356,17 +356,13 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
     util::Log() << "Expansion: " << nodes_per_second << " nodes/sec and " << edges_per_second
                 << " edges/sec";
     util::Log() << "To prepare the data for routing, run: "
-                << "./osrm-contract " << config.GetPath(".osrm");
+                << "./osrm-contract " << config.base_path;
 
     return 0;
 }
 
-std::tuple<LaneDescriptionMap,
+Extractor::ParsedOSMData Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
-           std::vector<TurnRestriction>,
+                                                 const unsigned number_of_threads)
-           std::vector<UnresolvedManeuverOverride>,
-           TrafficSignals>
-Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
-                        const unsigned number_of_threads)
 {
     TIMER_START(extracting);
 
@@ -617,7 +613,6 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
     }
 
     extraction_containers.PrepareData(scripting_environment,
-                                      config.GetPath(".osrm").string(),
                                       config.GetPath(".osrm.names").string());
 
     auto profile_properties = scripting_environment.GetProfileProperties();
@@ -629,10 +624,36 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
     TIMER_STOP(extracting);
     util::Log() << "extraction finished after " << TIMER_SEC(extracting) << "s";
 
-    return std::make_tuple(std::move(turn_lane_map),
+    std::vector<util::Coordinate> osm_coordinates;
-                           std::move(extraction_containers.turn_restrictions),
+    extractor::PackedOSMIDs osm_node_ids;
-                           std::move(extraction_containers.internal_maneuver_overrides),
+
-                           std::move(extraction_containers.internal_traffic_signals));
+    osm_coordinates.resize(extraction_containers.used_nodes.size());
+    osm_node_ids.reserve(extraction_containers.used_nodes.size());
+    for (size_t index = 0; index < extraction_containers.used_nodes.size(); ++index)
+    {
+        const auto &current_node = extraction_containers.used_nodes[index];
+        osm_coordinates[index].lon = current_node.lon;
+        osm_coordinates[index].lat = current_node.lat;
+        osm_node_ids.push_back(current_node.node_id);
+    }
+
+    if (config.dump_nbg_graph)
+    {
+        storage::tar::FileWriter writer(config.GetPath(".osrm.nbg").string(),
+                                        storage::tar::FileWriter::GenerateFingerprint);
+        storage::serialization::write(writer, "/extractor/nodes", extraction_containers.used_nodes);
+        storage::serialization::write(writer, "/extractor/edges", extraction_containers.used_edges);
+    }
+
+    return ParsedOSMData{std::move(turn_lane_map),
+                         std::move(extraction_containers.turn_restrictions),
+                         std::move(extraction_containers.internal_maneuver_overrides),
+                         std::move(extraction_containers.internal_traffic_signals),
+                         std::move(extraction_containers.used_barrier_nodes),
+                         std::move(osm_coordinates),
+                         std::move(osm_node_ids),
+                         std::move(extraction_containers.used_edges),
+                         std::move(extraction_containers.all_edges_annotation_data_list)};
 }
 
 void Extractor::FindComponents(unsigned number_of_edge_based_nodes,

src/extractor/node_based_graph_factory.cpp

@@ -16,32 +16,30 @@ namespace extractor
 {
 
 NodeBasedGraphFactory::NodeBasedGraphFactory(
-    const boost::filesystem::path &input_file,
     ScriptingEnvironment &scripting_environment,
     std::vector<TurnRestriction> &turn_restrictions,
     std::vector<UnresolvedManeuverOverride> &maneuver_overrides,
-    const TrafficSignals &traffic_signals)
+    const TrafficSignals &traffic_signals,
+    std::unordered_set<NodeID> &&barriers,
+    std::vector<util::Coordinate> &&coordinates,
+    extractor::PackedOSMIDs &&osm_node_ids,
+    const std::vector<NodeBasedEdge> &edge_list,
+    std::vector<NodeBasedEdgeAnnotation> &&annotation_data)
+    : annotation_data(std::move(annotation_data)), barriers(std::move(barriers)),
+      coordinates(std::move(coordinates)), osm_node_ids(std::move(osm_node_ids))
 {
-    LoadDataFromFile(input_file);
+    BuildCompressedOutputGraph(edge_list);
     Compress(scripting_environment, turn_restrictions, maneuver_overrides, traffic_signals);
     CompressGeometry();
     CompressAnnotationData();
 }
 
-// load the data serialised during the extraction run
+void NodeBasedGraphFactory::BuildCompressedOutputGraph(const std::vector<NodeBasedEdge> &edge_list)
-void NodeBasedGraphFactory::LoadDataFromFile(const boost::filesystem::path &input_file)
 {
-    auto barriers_iter = inserter(barriers, end(barriers));
-    std::vector<NodeBasedEdge> edge_list;
-
-    files::readRawNBGraph(
-        input_file, barriers_iter, coordinates, osm_node_ids, edge_list, annotation_data);
-
     const auto number_of_node_based_nodes = coordinates.size();
     if (edge_list.empty())
     {
-        throw util::exception("Node-based-graph (" + input_file.string() + ") contains no edges." +
+        throw util::exception("Node-based-graph contains no edges." + SOURCE_REF);
-                              SOURCE_REF);
     }
 
     // at this point, the data isn't compressed, but since we update the graph in-place, we assign

src/nodejs/node_osrm.cpp

@@ -55,8 +55,8 @@ NAN_MODULE_INIT(Engine::Init)
 // clang-format off
 /**
  * The `OSRM` method is the main constructor for creating an OSRM instance.
- * An OSRM instance requires a `.osrm` dataset, which is prepared by the OSRM toolchain.
+ * An OSRM instance requires a `.osrm.*` dataset(`.osrm.*` because it contains several files), which is prepared by the OSRM toolchain.
- * You can create such a `.osrm` file by running the OSRM binaries we ship in `node_modules/osrm/lib/binding/` and default
+ * You can create such a `.osrm.*` dataset by running the OSRM binaries we ship in `node_modules/osrm/lib/binding/` and default
  * profiles (e.g. for setting speeds and determining road types to route on) in `node_modules/osrm/profiles/`:
  *
  *     node_modules/osrm/lib/binding/osrm-extract data.osm.pbf -p node_modules/osrm/profiles/car.lua
@@ -64,7 +64,7 @@ NAN_MODULE_INIT(Engine::Init)
  *
  * Consult the [osrm-backend](https://github.com/Project-OSRM/osrm-backend) documentation for further details.
  *
- * Once you have a complete `network.osrm` file, you can calculate routes in javascript with this object.
+ * Once you have a complete `network.osrm.*` dataset, you can calculate routes in javascript with this object.
  *
  * ```javascript
  * var osrm = new OSRM('network.osrm');

src/tools/components.cpp

@@ -39,12 +39,8 @@ std::size_t loadGraph(const std::string &path,
                       std::vector<TarjanEdge> &graph_edge_list)
 {
     std::vector<extractor::NodeBasedEdge> edge_list;
-    std::vector<extractor::NodeBasedEdgeAnnotation> annotation_data;
 
-    auto nop = boost::make_function_output_iterator([](auto) {});
+    extractor::files::readRawNBGraph(path, coordinate_list, osm_node_ids, edge_list);
-
-    extractor::files::readRawNBGraph(
-        path, nop, coordinate_list, osm_node_ids, edge_list, annotation_data);
 
     // Building a node-based graph
     for (const auto &input_edge : edge_list)

src/tools/customize.cpp

@@ -77,7 +77,7 @@ return_code parseArguments(int argc,
     hidden_options.add_options()(
         "input,i",
         boost::program_options::value<boost::filesystem::path>(&customization_config.base_path),
-        "Input file in .osrm format");
+        "Input base file path");
 
     // positional option
     boost::program_options::positional_options_description positional_options;

src/tools/extract.cpp

@@ -74,7 +74,12 @@ return_code parseArguments(int argc,
         boost::program_options::bool_switch(&extractor_config.use_locations_cache)
             ->implicit_value(false)
             ->default_value(true),
-        "Use internal nodes locations cache for location-dependent data lookups");
+        "Use internal nodes locations cache for location-dependent data lookups")(
+        "dump-nbg-graph",
+        boost::program_options::bool_switch(&extractor_config.dump_nbg_graph)
+            ->implicit_value(true)
+            ->default_value(false),
+        "Dump raw node-based graph to *.osrm file for debug purposes.");
 
     bool dummy;
     // hidden options, will be allowed on command line, but will not be

src/tools/partition.cpp

@@ -119,7 +119,7 @@ return_code parseArguments(int argc,
     hidden_options.add_options()(
         "input,i",
         boost::program_options::value<boost::filesystem::path>(&config.base_path),
-        "Input file in .osrm format");
+        "Input base file path");
 
     // positional option
     boost::program_options::positional_options_description positional_options;

test/data/Makefile

@@ -22,15 +22,15 @@ clean:
 
 ch/$(DATA_NAME).osrm: $(DATA_NAME).osrm
 	mkdir -p ch
-	cp $(DATA_NAME).osrm $(DATA_NAME).osrm.* ch/
+	cp $(DATA_NAME).osrm.* ch/
 
 corech/$(DATA_NAME).osrm: $(DATA_NAME).osrm
 	mkdir -p corech
-	cp $(DATA_NAME).osrm $(DATA_NAME).osrm.* corech/
+	cp $(DATA_NAME).osrm.* corech/
 
 mld/$(DATA_NAME).osrm: $(DATA_NAME).osrm
 	mkdir -p mld
-	cp $(DATA_NAME).osrm $(DATA_NAME).osrm.* mld/
+	cp $(DATA_NAME).osrm.* mld/
 
 $(DATA_NAME).osrm: $(DATA_NAME).osm.pbf $(DATA_NAME).poly $(PROFILE) $(OSRM_EXTRACT)
 	@echo "Verifiyng data file integrity..."