Better statistics for osrm-partition and osrm-custimize

include/partitioner/cell_statistics.hpp

@@ -0,0 +1,54 @@
+#ifndef OSRM_PARTITIONER_CELL_STATISTICS_HPP
+#define OSRM_PARTITIONER_CELL_STATISTICS_HPP
+
+#include "util/log.hpp"
+#include "util/typedefs.hpp"
+
+#include <unordered_set>
+
+namespace osrm
+{
+namespace partitioner
+{
+template <typename Partition, typename CellStorage>
+void printCellStatistics(const Partition &partition, const CellStorage &storage)
+{
+    util::Log() << "Cells statistics per level";
+
+    for (std::size_t level = 1; level < partition.GetNumberOfLevels(); ++level)
+    {
+        auto num_cells = partition.GetNumberOfCells(level);
+        std::size_t source = 0, destination = 0;
+        std::size_t boundary_nodes = 0;
+        std::size_t entries = 0;
+        for (std::uint32_t cell_id = 0; cell_id < num_cells; ++cell_id)
+        {
+            std::unordered_set<NodeID> boundary;
+            const auto &cell = storage.GetUnfilledCell(level, cell_id);
+            source += cell.GetSourceNodes().size();
+            destination += cell.GetDestinationNodes().size();
+            for (auto node : cell.GetSourceNodes())
+            {
+                boundary.insert(node);
+            }
+            for (auto node : cell.GetDestinationNodes())
+            {
+                boundary.insert(node);
+            }
+            boundary_nodes += boundary.size();
+            entries += cell.GetSourceNodes().size() * cell.GetDestinationNodes().size();
+        }
+
+        source /= num_cells;
+        destination /= num_cells;
+
+        util::Log() << "Level " << level << " #cells " << num_cells << " #boundary nodes "
+                    << boundary_nodes << ", sources: avg. " << source << ", destinations: avg. "
+                    << destination << ", entries: " << entries << " ("
+                    << (2 * entries * sizeof(EdgeWeight)) << " bytes)";
+    }
+}
+}
+}
+
+#endif

include/partitioner/cell_storage.hpp

@@ -193,6 +193,20 @@ template <storage::Ownership Ownership> class CellStorageImpl
             BOOST_ASSERT(num_source_nodes == 0 || all_sources != nullptr);
             BOOST_ASSERT(num_destination_nodes == 0 || all_destinations != nullptr);
         }
+
+        // Consturcts an emptry cell without weights. Useful when only access
+        // to the cell structure is needed, without a concrete metric.
+        CellImpl(const CellData &data,
+                 const NodeID *const all_sources,
+                 const NodeID *const all_destinations)
+            : num_source_nodes{data.num_source_nodes},
+              num_destination_nodes{data.num_destination_nodes}, weights{nullptr},
+              durations{nullptr}, source_boundary{all_sources + data.source_boundary_offset},
+              destination_boundary{all_destinations + data.destination_boundary_offset}
+        {
+            BOOST_ASSERT(num_source_nodes == 0 || all_sources != nullptr);
+            BOOST_ASSERT(num_destination_nodes == 0 || all_destinations != nullptr);
+        }
     };
 
     std::size_t LevelIDToIndex(LevelID level) const { return level - 1; }
@@ -378,6 +392,18 @@ template <storage::Ownership Ownership> class CellStorageImpl
                          destination_boundary.empty() ? nullptr : destination_boundary.data()};
     }
 
+    ConstCell GetUnfilledCell(LevelID level, CellID id) const
+    {
+        const auto level_index = LevelIDToIndex(level);
+        BOOST_ASSERT(level_index < level_to_cell_offset.size());
+        const auto offset = level_to_cell_offset[level_index];
+        const auto cell_index = offset + id;
+        BOOST_ASSERT(cell_index < cells.size());
+        return ConstCell{cells[cell_index],
+                         source_boundary.empty() ? nullptr : source_boundary.data(),
+                         destination_boundary.empty() ? nullptr : destination_boundary.data()};
+    }
+
     template <typename = std::enable_if<Ownership == storage::Ownership::Container>>
     Cell GetCell(customizer::CellMetric &metric, LevelID level, CellID id) const
     {

src/customize/customizer.cpp

@@ -5,6 +5,7 @@
 #include "customizer/edge_based_graph.hpp"
 #include "customizer/files.hpp"
 
+#include "partitioner/cell_statistics.hpp"
 #include "partitioner/cell_storage.hpp"
 #include "partitioner/edge_based_graph_reader.hpp"
 #include "partitioner/files.hpp"
@@ -29,30 +30,22 @@ namespace customizer
 
 namespace
 {
-template <typename Graph, typename Partition, typename CellStorage>
+
-void CellStorageStatistics(const Graph &graph,
+template <typename Partition, typename CellStorage>
-                           const Partition &partition,
+void printUnreachableStatistics(const Partition &partition,
-                           const CellStorage &storage,
+                                const CellStorage &storage,
-                           const CellMetric &metric)
+                                const CellMetric &metric)
 {
-    util::Log() << "Cells statistics per level";
+    util::Log() << "Unreachable nodes statistics per level";
 
     for (std::size_t level = 1; level < partition.GetNumberOfLevels(); ++level)
     {
-        std::unordered_map<CellID, std::size_t> cell_nodes;
+        auto num_cells = partition.GetNumberOfCells(level);
-        for (auto node : util::irange(0u, graph.GetNumberOfNodes()))
+        std::size_t invalid_sources = 0;
-        {
+        std::size_t invalid_destinations = 0;
-            ++cell_nodes[partition.GetCell(level, node)];
+        for (std::uint32_t cell_id = 0; cell_id < num_cells; ++cell_id)
-        }
-
-        std::size_t source = 0, destination = 0, total = 0;
-        std::size_t invalid_sources = 0, invalid_destinations = 0;
-        for (std::uint32_t cell_id = 0; cell_id < partition.GetNumberOfCells(level); ++cell_id)
         {
             const auto &cell = storage.GetCell(metric, level, cell_id);
-            source += cell.GetSourceNodes().size();
-            destination += cell.GetDestinationNodes().size();
-            total += cell_nodes[cell_id];
             for (auto node : cell.GetSourceNodes())
             {
                 const auto &weights = cell.GetOutWeight(node);
@@ -70,15 +63,12 @@ void CellStorageStatistics(const Graph &graph,
             }
         }
 
-        util::Log() << "Level " << level << " #cells " << cell_nodes.size() << " #nodes " << total
+        if (invalid_sources > 0 || invalid_destinations > 0)
-                    << ",   source nodes: average " << source << " (" << (100. * source / total)
+        {
-                    << "%)"
+            util::Log(logWARNING) << "Level " << level << " unreachable boundary nodes per cell: "
-                    << " invalid " << invalid_sources << " (" << (100. * invalid_sources / total)
+                                  << (invalid_sources / (float)num_cells) << " sources, "
-                    << "%)"
+                                  << (invalid_destinations / (float)num_cells) << " destinations";
-                    << ",   destination nodes: average " << destination << " ("
+        }
-                    << (100. * destination / total) << "%)"
-                    << " invalid " << invalid_destinations << " ("
-                    << (100. * invalid_destinations / total) << "%)";
     }
 }
 
@@ -152,6 +142,12 @@ int Customizer::Run(const CustomizationConfig &config)
     TIMER_STOP(cell_customize);
     util::Log() << "Cells customization took " << TIMER_SEC(cell_customize) << " seconds";
 
+    partitioner::printCellStatistics(mlp, storage);
+    for (const auto &metric : metrics)
+    {
+        printUnreachableStatistics(mlp, storage, metric);
+    }
+
     TIMER_START(writing_mld_data);
     std::unordered_map<std::string, std::vector<CellMetric>> metric_exclude_classes = {
         {properties.GetWeightName(), std::move(metrics)},
@@ -165,11 +161,6 @@ int Customizer::Run(const CustomizationConfig &config)
     TIMER_STOP(writing_graph);
     util::Log() << "Graph writing took " << TIMER_SEC(writing_graph) << " seconds";
 
-    for (const auto &metric : metrics)
-    {
-        CellStorageStatistics(graph, mlp, storage, metric);
-    }
-
     return 0;
 }
 

src/partitioner/partitioner.cpp

@@ -1,6 +1,7 @@
 #include "partitioner/partitioner.hpp"
 #include "partitioner/bisection_graph.hpp"
 #include "partitioner/bisection_to_partition.hpp"
+#include "partitioner/cell_statistics.hpp"
 #include "partitioner/cell_storage.hpp"
 #include "partitioner/edge_based_graph_reader.hpp"
 #include "partitioner/files.hpp"
@@ -181,6 +182,8 @@ int Partitioner::Run(const PartitionerConfig &config)
     TIMER_STOP(writing_mld_data);
     util::Log() << "MLD data writing took " << TIMER_SEC(writing_mld_data) << " seconds";
 
+    printCellStatistics(mlp, storage);
+
     return 0;
 }