Do not generate intermediate .osrm file in osrm-extract.

.github/workflows/osrm-backend.yml

@@ -108,12 +108,16 @@ jobs:
           rm -rf "${PWD}/berlin-latest.osrm"
           docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-partition /data/berlin-latest.osrm
 
-          docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-partition /data/berlin-latest
+          docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-partition /data/berlin-latest.osm.pbf
-          docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-customize /data/berlin-latest
+          docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-customize /data/berlin-latest.osm.pbf
-          docker run $MEMORY_ARGS --name=osrm-container -t -p 5000:5000 -v "${PWD}:/data" "${TAG}" osrm-routed --algorithm mld /data/berlin-latest &
+          docker run $MEMORY_ARGS --name=osrm-container -t -p 5000:5000 -v "${PWD}:/data" "${TAG}" osrm-routed --algorithm mld /data/berlin-latest.osm.pbf &
           curl --retry-delay 3 --retry 10 --retry-all-errors "http://127.0.0.1:5000/route/v1/driving/13.388860,52.517037;13.385983,52.496891?steps=true"
           docker stop osrm-container
 
+          # check that we accept base file path(without `.osm.pbf` extension)
+          docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-partition /data/berlin-latest
+          docker run $MEMORY_ARGS -t -v "${PWD}:/data" "${TAG}" osrm-customize /data/berlin-latest
+
   build-test-publish:
     needs: format-taginfo-docs
     strategy:

README.md

@@ -62,12 +62,12 @@ Pre-process the extract with the car profile and start a routing engine HTTP ser
 
 The flag `-v "${PWD}:/data"` creates the directory `/data` inside the docker container and makes the current working directory `"${PWD}"` available there. The file `/data/berlin-latest.osm.pbf` inside the container is referring to `"${PWD}/berlin-latest.osm.pbf"` on the host.
 
-    docker run -t -v "${PWD}:/data" osrm/osrm-backend osrm-partition /data/berlin-latest
+    docker run -t -v "${PWD}:/data" osrm/osrm-backend osrm-partition /data/berlin-latest.osm.pbf
-    docker run -t -v "${PWD}:/data" osrm/osrm-backend osrm-customize /data/berlin-latest
+    docker run -t -v "${PWD}:/data" osrm/osrm-backend osrm-customize /data/berlin-latest.osm.pbf
 
-Note that `berlin-latest` has no file extension. 
+Note that only `osrm-extract` actually uses input `.osm.pbf`. All other tools work on top of `osrm-extract` output and when `.osm.pbf` is passed to the tool it is only needed to determine "base" path and find output of `osrm-extract`, i.e. these tools will also work if just pass "base" file path(e.g. `osrm-partition /data/berlin-latest`). 
 
-    docker run -t -i -p 5000:5000 -v "${PWD}:/data" osrm/osrm-backend osrm-routed --algorithm mld /data/berlin-latest
+    docker run -t -i -p 5000:5000 -v "${PWD}:/data" osrm/osrm-backend osrm-routed --algorithm mld /data/berlin-latest.osm.pbf
 
 Make requests against the HTTP server
 

include/extractor/extraction_containers.hpp

@@ -72,7 +72,7 @@ class ExtractionContainers
     std::vector<InputTrafficSignal> external_traffic_signals;
     TrafficSignals internal_traffic_signals;
 
-    std::vector<NodeBasedEdge> normal_edges;
+    std::vector<NodeBasedEdge> used_edges;
 
     // List of restrictions (conditional and unconditional) before we transform them into the
     // output types. Input containers reference OSMNodeIDs. We can only transform them to the
@@ -83,8 +83,8 @@ class ExtractionContainers
 
     std::vector<InputManeuverOverride> external_maneuver_overrides_list;
     std::vector<UnresolvedManeuverOverride> internal_maneuver_overrides;
-    std::unordered_set<NodeID> internal_barrier_nodes;
+    std::unordered_set<NodeID> used_barrier_nodes;
-    NodeVector internal_nodes;
+    NodeVector used_nodes;
 
     ExtractionContainers();
 

include/extractor/node_based_graph_factory.hpp

@@ -34,7 +34,7 @@ namespace extractor
 class NodeBasedGraphFactory
 {
   public:
-    // The node-based graph factory loads the *.osrm file and transforms the data within into the
+    // The node-based graph factory transforms the graph data into the
     // node-based graph to represent the OSM network. This includes geometry compression, annotation
     // data optimisation and many other aspects. After this step, the edge-based graph factory can
     // turn the graph into the routing graph to be used with the navigation algorithms.

src/extractor/extraction_containers.cpp

@@ -542,7 +542,7 @@ void ExtractionContainers::PrepareNodes()
 
             ++node_id_iterator;
 
-            internal_nodes.emplace_back(*node_iterator++);
+            used_nodes.emplace_back(*node_iterator++);
         }
 
         TIMER_STOP(write_nodes);
@@ -559,7 +559,7 @@ void ExtractionContainers::PrepareNodes()
                 used_node_id_list.begin(), used_node_id_list.end(), osm_id);
             if (node_id != SPECIAL_NODEID)
             {
-                internal_barrier_nodes.emplace(node_id);
+                used_barrier_nodes.emplace(node_id);
             }
         }
         log << "ok, after " << TIMER_SEC(write_nodes) << "s";
@@ -852,10 +852,10 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
         }
     }
 
-    all_nodes_list.clear(); // free all_nodes_list before allocation of normal_edges
+    all_nodes_list.clear(); // free all_nodes_list before allocation of used_edges
     all_nodes_list.shrink_to_fit();
 
-    normal_edges.reserve(all_edges_list.size());
+    used_edges.reserve(all_edges_list.size());
     {
         util::UnbufferedLog log;
         log << "Writing used edges       ... " << std::flush;
@@ -871,17 +871,17 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm
 
             // IMPORTANT: here, we're using slicing to only write the data from the base
             // class of NodeBasedEdgeWithOSM
-            normal_edges.push_back(edge.result);
+            used_edges.push_back(edge.result);
         }
 
-        if (normal_edges.size() > std::numeric_limits<uint32_t>::max())
+        if (used_edges.size() > std::numeric_limits<uint32_t>::max())
         {
             throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF);
         }
 
         TIMER_STOP(write_edges);
         log << "ok, after " << TIMER_SEC(write_edges) << "s";
-        log << " -- Processed " << normal_edges.size() << " edges";
+        log << " -- Processed " << used_edges.size() << " edges";
     }
 }
 

src/extractor/extractor.cpp

@@ -627,11 +627,11 @@ Extractor::ParsedOSMData Extractor::ParseOSMData(ScriptingEnvironment &scripting
     std::vector<util::Coordinate> osm_coordinates;
     extractor::PackedOSMIDs osm_node_ids;
 
-    osm_coordinates.resize(extraction_containers.internal_nodes.size());
+    osm_coordinates.resize(extraction_containers.used_nodes.size());
-    osm_node_ids.reserve(extraction_containers.internal_nodes.size());
+    osm_node_ids.reserve(extraction_containers.used_nodes.size());
-    for (size_t index = 0; index < extraction_containers.internal_nodes.size(); ++index)
+    for (size_t index = 0; index < extraction_containers.used_nodes.size(); ++index)
     {
-        const auto &current_node = extraction_containers.internal_nodes[index];
+        const auto &current_node = extraction_containers.used_nodes[index];
         osm_coordinates[index].lon = current_node.lon;
         osm_coordinates[index].lat = current_node.lat;
         osm_node_ids.push_back(current_node.node_id);
@@ -642,19 +642,19 @@ Extractor::ParsedOSMData Extractor::ParseOSMData(ScriptingEnvironment &scripting
         storage::tar::FileWriter writer(config.GetPath(".osrm").string(),
                                         storage::tar::FileWriter::GenerateFingerprint);
         storage::serialization::write(
-            writer, "/extractor/nodes", extraction_containers.internal_nodes);
+            writer, "/extractor/nodes", extraction_containers.used_nodes);
         storage::serialization::write(
-            writer, "/extractor/edges", extraction_containers.normal_edges);
+            writer, "/extractor/edges", extraction_containers.used_edges);
     }
 
     return ParsedOSMData{std::move(turn_lane_map),
                          std::move(extraction_containers.turn_restrictions),
                          std::move(extraction_containers.internal_maneuver_overrides),
                          std::move(extraction_containers.internal_traffic_signals),
-                         std::move(extraction_containers.internal_barrier_nodes),
+                         std::move(extraction_containers.used_barrier_nodes),
                          std::move(osm_coordinates),
                          std::move(osm_node_ids),
-                         std::move(extraction_containers.normal_edges),
+                         std::move(extraction_containers.used_edges),
                          std::move(extraction_containers.all_edges_annotation_data_list)};
 }
 

src/tools/customize.cpp

@@ -77,7 +77,7 @@ return_code parseArguments(int argc,
     hidden_options.add_options()(
         "input,i",
         boost::program_options::value<boost::filesystem::path>(&customization_config.base_path),
-        "Input file in .osrm format");
+        "Input base file path");
 
     // positional option
     boost::program_options::positional_options_description positional_options;
@@ -89,7 +89,7 @@ return_code parseArguments(int argc,
 
     const auto *executable = argv[0];
     boost::program_options::options_description visible_options(
-        boost::filesystem::path(executable).filename().string() + " <input.osrm> [options]");
+        boost::filesystem::path(executable).filename().string() + " <input.osm.pbf> [options]");
     visible_options.add(generic_options).add(config_options);
 
     // parse command line options

src/tools/partition.cpp

@@ -119,7 +119,7 @@ return_code parseArguments(int argc,
     hidden_options.add_options()(
         "input,i",
         boost::program_options::value<boost::filesystem::path>(&config.base_path),
-        "Input file in .osrm format");
+        "Input base file path");
 
     // positional option
     boost::program_options::positional_options_description positional_options;
@@ -131,7 +131,7 @@ return_code parseArguments(int argc,
 
     const auto *executable = argv[0];
     boost::program_options::options_description visible_options(
-        boost::filesystem::path(executable).filename().string() + " <input.osrm> [options]");
+        boost::filesystem::path(executable).filename().string() + " <input.osm.pbf> [options]");
     visible_options.add(generic_options).add(config_options);
 
     // parse command line options