Renumber nodes after running osrm-partition

The new numbering uses the partition information
to sort border nodes first to compactify storages
that need access indexed by border node ID.

We also get an optimized cache performance for free
sincr we can also recursively sort the nodes by cell ID.

This implements issue #3779.

src/extractor/extractor.cpp

@@ -178,7 +178,13 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
     files::writeNodes(config.node_based_nodes_data_path, coordinates, osm_node_ids);
     files::writeNodeData(config.edge_based_nodes_data_path, edge_based_nodes_container);
 
-    WriteEdgeBasedGraph(config.edge_graph_output_path, max_edge_id, edge_based_edge_list);
+    util::Log() << "Writing edge-based-graph edges       ... " << std::flush;
+    TIMER_START(write_edges);
+    files::writeEdgeBasedGraph(config.edge_graph_output_path, max_edge_id, edge_based_edge_list);
+    TIMER_STOP(write_edges);
+    util::Log() << "ok, after " << TIMER_SEC(write_edges) << "s";
+
+    util::Log() << "Processed " << edge_based_edge_list.size() << " edges";
 
     const auto nodes_per_second =
         static_cast<std::uint64_t>(number_of_node_based_nodes / TIMER_SEC(expansion));
@@ -578,32 +584,6 @@ void Extractor::BuildRTree(std::vector<EdgeBasedNodeSegment> edge_based_node_seg
     util::Log() << "finished r-tree construction in " << TIMER_SEC(construction) << " seconds";
 }
 
-void Extractor::WriteEdgeBasedGraph(
-    std::string const &output_file_filename,
-    EdgeID const max_edge_id,
-    util::DeallocatingVector<EdgeBasedEdge> const &edge_based_edge_list)
-{
-    storage::io::FileWriter file(output_file_filename,
-                                 storage::io::FileWriter::GenerateFingerprint);
-
-    util::Log() << "Writing edge-based-graph edges       ... " << std::flush;
-    TIMER_START(write_edges);
-
-    std::uint64_t number_of_used_edges = edge_based_edge_list.size();
-    file.WriteElementCount64(number_of_used_edges);
-    file.WriteOne(max_edge_id);
-
-    for (const auto &edge : edge_based_edge_list)
-    {
-        file.WriteOne(edge);
-    }
-
-    TIMER_STOP(write_edges);
-    util::Log() << "ok, after " << TIMER_SEC(write_edges) << "s";
-
-    util::Log() << "Processed " << number_of_used_edges << " edges";
-}
-
 void Extractor::WriteIntersectionClassificationData(
     const std::string &output_file_name,
     const std::vector<BearingClassID> &node_based_intersection_classes,

src/partition/partitioner.cpp

@@ -8,6 +8,7 @@
 #include "partition/multi_level_partition.hpp"
 #include "partition/recursive_bisection.hpp"
 #include "partition/remove_unconnected.hpp"
+#include "partition/renumber.hpp"
 
 #include "extractor/files.hpp"
 
@@ -17,12 +18,14 @@
 #include "util/integer_range.hpp"
 #include "util/json_container.hpp"
 #include "util/log.hpp"
+#include "util/mmap_file.hpp"
 
 #include <algorithm>
 #include <iterator>
 #include <vector>
 
 #include <boost/assert.hpp>
+#include <boost/filesystem/operations.hpp>
 
 #include "util/geojson_debug_logger.hpp"
 #include "util/geojson_debug_policies.hpp"
@@ -129,10 +132,10 @@ int Partitioner::Run(const PartitionConfig &config)
     extractor::files::readNBGMapping(config.cnbg_ebg_mapping_path.string(), mapping);
     util::Log() << "Loaded node based graph to edge based graph mapping";
 
-    auto edge_based_graph = LoadEdgeBasedGraph(config.edge_based_graph_path.string());
+    auto edge_based_graph = LoadEdgeBasedGraph(config.edge_based_graph_path);
     util::Log() << "Loaded edge based graph for mapping partition ids: "
-                << edge_based_graph->GetNumberOfEdges() << " edges, "
-                << edge_based_graph->GetNumberOfNodes() << " nodes";
+                << edge_based_graph.GetNumberOfEdges() << " edges, "
+                << edge_based_graph.GetNumberOfNodes() << " nodes";
 
     // TODO: node based graph to edge based graph partition id mapping should be done split off.
 
@@ -140,7 +143,7 @@ int Partitioner::Run(const PartitionConfig &config)
     const auto &node_based_partition_ids = recursive_bisection.BisectionIDs();
 
     // Partition ids, keyed by edge based graph nodes
-    std::vector<NodeID> edge_based_partition_ids(edge_based_graph->GetNumberOfNodes(),
+    std::vector<NodeID> edge_based_partition_ids(edge_based_graph.GetNumberOfNodes(),
                                                  SPECIAL_NODEID);
 
     // Only resolve all easy cases in the first pass
@@ -163,7 +166,7 @@ int Partitioner::Run(const PartitionConfig &config)
     std::tie(partitions, level_to_num_cells) =
         bisectionToPartition(edge_based_partition_ids, config.max_cell_sizes);
 
-    auto num_unconnected = removeUnconnectedBoundaryNodes(*edge_based_graph, partitions);
+    auto num_unconnected = removeUnconnectedBoundaryNodes(edge_based_graph, partitions);
     util::Log() << "Fixed " << num_unconnected << " unconnected nodes";
 
     util::Log() << "Edge-based-graph annotation:";
@@ -173,19 +176,47 @@ int Partitioner::Run(const PartitionConfig &config)
                     << " bit size " << std::ceil(std::log2(level_to_num_cells[level] + 1));
     }
 
+    TIMER_START(renumber);
+    auto permutation = makePermutation(edge_based_graph, partitions);
+    renumber(edge_based_graph, permutation);
+    renumber(partitions, permutation);
+    {
+        boost::iostreams::mapped_file segment_region;
+        auto segments =
+            util::mmapFile<extractor::EdgeBasedNodeSegment>(config.file_index_path, segment_region);
+        renumber(segments, permutation);
+    }
+    {
+        extractor::EdgeBasedNodeDataContainer node_data;
+        extractor::files::readNodeData(config.node_data_path, node_data);
+        renumber(node_data, permutation);
+        extractor::files::writeNodeData(config.node_data_path, node_data);
+    }
+    if (boost::filesystem::exists(config.hsgr_path))
+    {
+        util::Log(logWARNING) << "Found existing .osrm.hsgr file, removing. You need to re-run "
+                                 "osrm-contract after osrm-partition.";
+        boost::filesystem::remove(config.hsgr_path);
+    }
+    TIMER_STOP(renumber);
+    util::Log() << "Renumbered data in " << TIMER_SEC(renumber) << " seconds";
+
     TIMER_START(packed_mlp);
     MultiLevelPartition mlp{partitions, level_to_num_cells};
     TIMER_STOP(packed_mlp);
     util::Log() << "MultiLevelPartition constructed in " << TIMER_SEC(packed_mlp) << " seconds";
 
     TIMER_START(cell_storage);
-    CellStorage storage(mlp, *edge_based_graph);
+    CellStorage storage(mlp, edge_based_graph);
     TIMER_STOP(cell_storage);
     util::Log() << "CellStorage constructed in " << TIMER_SEC(cell_storage) << " seconds";
 
     TIMER_START(writing_mld_data);
-    files::writePartition(config.mld_partition_path, mlp);
-    files::writeCells(config.mld_storage_path, storage);
+    files::writePartition(config.partition_path, mlp);
+    files::writeCells(config.storage_path, storage);
+    extractor::files::writeEdgeBasedGraph(config.edge_based_graph_path,
+                                          edge_based_graph.GetNumberOfNodes() - 1,
+                                          graphToEdges(edge_based_graph));
     TIMER_STOP(writing_mld_data);
     util::Log() << "MLD data writing took " << TIMER_SEC(writing_mld_data) << " seconds";
 

src/partition/renumber.cpp

@@ -0,0 +1,74 @@
+#include "partition/renumber.hpp"
+
+#include <tbb/parallel_sort.h>
+
+namespace osrm
+{
+namespace partition
+{
+namespace
+{
+// Returns a vector that is indexed by node ID marking the level at which it is a border node
+std::vector<LevelID> getHighestBorderLevel(const DynamicEdgeBasedGraph &graph,
+                                           const std::vector<Partition> &partitions)
+{
+    std::vector<LevelID> border_level(graph.GetNumberOfNodes(), 0);
+
+    for (const auto level : util::irange<LevelID>(1, partitions.size() + 1))
+    {
+        const auto &partition = partitions[level - 1];
+        for (auto node : util::irange<NodeID>(0, graph.GetNumberOfNodes()))
+        {
+            for (auto edge : graph.GetAdjacentEdgeRange(node))
+            {
+                auto target = graph.GetTarget(edge);
+                if (partition[node] != partition[target])
+                {
+                    // level is monotone increasing so we wil
+                    // always overwrite here with a value equal
+                    // or greater then the current border_level
+                    border_level[node] = level;
+                    border_level[target] = level;
+                }
+            }
+        }
+    }
+
+    return border_level;
+}
+}
+
+std::vector<std::uint32_t> makePermutation(const DynamicEdgeBasedGraph &graph,
+                                           const std::vector<Partition> &partitions)
+{
+    std::vector<std::uint32_t> ordering(graph.GetNumberOfNodes());
+    std::iota(ordering.begin(), ordering.end(), 0);
+
+    // Sort the nodes by cell ID recursively:
+    // Nodes in the same cell will be sorted by cell ID on the level below
+    for (const auto &partition : partitions)
+    {
+        std::stable_sort(
+            ordering.begin(), ordering.end(), [&partition](const auto lhs, const auto rhs) {
+                return partition[lhs] < partition[rhs];
+            });
+    }
+
+    // Now sort the nodes by the level at which they are a border node, descening.
+    // That means nodes that are border nodes on the highest level will have a very low ID,
+    // whereas nodes that are nerver border nodes are sorted to the end of the array.
+    // Note: Since we use a stable sort that preserves the cell sorting within each level
+    auto border_level = getHighestBorderLevel(graph, partitions);
+    std::stable_sort(
+        ordering.begin(), ordering.end(), [&border_level](const auto lhs, const auto rhs) {
+            return border_level[lhs] > border_level[rhs];
+        });
+
+    std::vector<std::uint32_t> permutation(ordering.size());
+    for (auto index : util::irange<std::uint32_t>(0, ordering.size()))
+        permutation[ordering[index]] = index;
+
+    return permutation;
+}
+}
+}

src/updater/updater.cpp

@@ -142,26 +142,6 @@ void checkWeightsConsistency(
 }
 #endif
 
-auto mmapFile(const std::string &filename, boost::interprocess::mode_t mode)
-{
-    using boost::interprocess::file_mapping;
-    using boost::interprocess::mapped_region;
-
-    try
-    {
-        const file_mapping mapping{filename.c_str(), mode};
-
-        mapped_region region{mapping, mode};
-        region.advise(mapped_region::advice_sequential);
-        return region;
-    }
-    catch (const std::exception &e)
-    {
-        util::Log(logERROR) << "Error while trying to mmap " + filename + ": " + e.what();
-        throw;
-    }
-}
-
 tbb::concurrent_vector<GeometryID>
 updateSegmentData(const UpdaterConfig &config,
                   const extractor::ProfileProperties &profile_properties,
@@ -420,14 +400,11 @@ updateTurnPenalties(const UpdaterConfig &config,
                     extractor::PackedOSMIDs osm_node_ids)
 {
     const auto weight_multiplier = profile_properties.GetWeightMultiplier();
-    const auto turn_index_region =
-        mmapFile(config.turn_penalties_index_path, boost::interprocess::read_only);
 
     // Mapped file pointer for turn indices
-    const extractor::lookup::TurnIndexBlock *turn_index_blocks =
-        reinterpret_cast<const extractor::lookup::TurnIndexBlock *>(
-            turn_index_region.get_address());
-    BOOST_ASSERT(is_aligned<extractor::lookup::TurnIndexBlock>(turn_index_blocks));
+    boost::iostreams::mapped_file_source turn_index_region;
+    auto turn_index_blocks = util::mmapFile<extractor::lookup::TurnIndexBlock>(
+        config.turn_penalties_index_path, turn_index_region);
 
     // Get the turn penalty and update to the new value if required
     std::vector<std::uint64_t> updated_turns;
@@ -508,13 +485,10 @@ updateConditionalTurns(const UpdaterConfig &config,
                        extractor::PackedOSMIDs &osm_node_ids,
                        Timezoner time_zone_handler)
 {
-    const auto turn_index_region =
-        mmapFile(config.turn_penalties_index_path, boost::interprocess::read_only);
     // Mapped file pointer for turn indices
-    const extractor::lookup::TurnIndexBlock *turn_index_blocks =
-        reinterpret_cast<const extractor::lookup::TurnIndexBlock *>(
-            turn_index_region.get_address());
-    BOOST_ASSERT(is_aligned<extractor::lookup::TurnIndexBlock>(turn_index_blocks));
+    boost::iostreams::mapped_file_source turn_index_region;
+    auto turn_index_blocks = util::mmapFile<extractor::lookup::TurnIndexBlock>(
+        config.turn_penalties_index_path, turn_index_region);
 
     std::vector<std::uint64_t> updated_turns;
     if (conditional_turns.size() == 0)
@@ -589,16 +563,9 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
     std::vector<util::Coordinate> coordinates;
     extractor::PackedOSMIDs osm_node_ids;
 
-    {
-        storage::io::FileReader reader(config.edge_based_graph_path,
-                                       storage::io::FileReader::VerifyFingerprint);
-        auto num_edges = reader.ReadElementCount64();
-        edge_based_edge_list.resize(num_edges);
-        max_edge_id = reader.ReadOne<EdgeID>();
-        reader.ReadInto(edge_based_edge_list);
-
-        extractor::files::readNodes(config.node_based_nodes_data_path, coordinates, osm_node_ids);
-    }
+    extractor::files::readEdgeBasedGraph(
+        config.edge_based_graph_path, max_edge_id, edge_based_edge_list);
+    extractor::files::readNodes(config.node_based_nodes_data_path, coordinates, osm_node_ids);
 
     const bool update_conditional_turns =
         !config.turn_restrictions_path.empty() && config.valid_now;

src/util/timed_historgram.cpp

@@ -0,0 +1,12 @@
+#include "util/timed_histogram.hpp"
+
+namespace osrm
+{
+namespace util
+{
+namespace detail
+{
+std::atomic_uint operation = {0};
+}
+}
+}