generate stats in annotation

This commit is contained in:
Moritz Kobitzsch 2017-02-07 10:26:29 +01:00 committed by Patrick Niklaus
parent 739ad73ae9
commit be41e8b321
14 changed files with 639 additions and 200 deletions

View File

@ -0,0 +1,136 @@
#ifndef OSRM_PARTITION_ANNOTATE_HPP_
#define OSRM_PARTITION_ANNOTATE_HPP_
#include "partition/bisection_graph.hpp"
#include "util/typedefs.hpp"
#include <cstdint>
#include <utility>
#include <vector>
namespace osrm
{
namespace partition
{
// takes the result of a recursive bisection and turns it into an annotated partition for MLD. These
// annotated partitions provide a mapping from every node in the graph to a consecutively
// numbered cell in each level of the multi level partition. Instead of using the bisection directly
// (which can result in a unbalanced tree structure)
// 
// _____o______
// / \
// o ____o____
// / \ / \
// a b o _o_
// / \ / \
// c d o o
// / \ / \
// e f g h
// 
// we build a balanced structure that will result in a multi-cut on any level. We transform this
// layout into:
//
// _____o__________
// / | \
// o | \
// / \ | \
// a b o _o_
// / \ / \
// c d o o
// / \ / \
// e f g h
class AnnotatedPartition
{
public:
// Used to generate an implicit tree representation
struct SizedID
{
BisectionID id;
std::size_t count;
bool operator<(const SizedID &other) const { return id < other.id; };
};
// Metrics that describe a single level
struct LevelMetrics
{
std::size_t border_nodes;
std::size_t border_arcs;
// impresses imbalance, if not all nodes are in that cell anymore
std::size_t contained_nodes;
std::size_t number_of_cells;
std::size_t max_border_nodes_per_cell;
std::size_t max_border_arcs_per_cell;
std::size_t total_memory_cells;
std::vector<std::size_t> cell_sizes;
std::ostream &print(std::ostream &os) const
{
os << "[level]\n"
<< "\t#border nodes: " << border_nodes << " #border arcs: " << border_arcs
<< " #cells: " << number_of_cells << " #contained nodes: " << contained_nodes << "\n"
<< "\tborder nodes: max: " << max_border_nodes_per_cell
<< " avg : " << static_cast<double>(border_nodes) / number_of_cells
<< " border arcs: max: " << max_border_arcs_per_cell << " "
<< " avg: " << static_cast<double>(border_arcs) / number_of_cells << "\n"
<< "\tmemory consumption: " << total_memory_cells / (1024.0 * 1024.0) << " MB."
<< "\n";
os << "\tcell sizes:";
for (auto s : cell_sizes)
os << " " << s;
os << std::endl;
return os;
}
std::ostream &logMachinereadable(std::ostream &os,
const std::string &identification,
std::size_t depth,
const bool print_header = false) const
{
if (print_header)
os << "[" << identification << "] # depth cells total_nodes border_nodes "
"max_border_nodes border_arcs max_border_arcs bytes "
"cell_sizes*\n";
os << "[" << identification << "] " << depth << " " << number_of_cells << " "
<< contained_nodes << " " << border_nodes << " " << max_border_nodes_per_cell << " "
<< border_arcs << " " << max_border_arcs_per_cell << " " << total_memory_cells;
for (auto s : cell_sizes)
os << " " << s;
os << "\n";
return os;
}
};
AnnotatedPartition(const BisectionGraph &graph, const std::vector<BisectionID> &bisection_ids);
private:
// print distribution of level graph as it is
void PrintBisection(const std::vector<SizedID> &implicit_tree,
const BisectionGraph &graph,
const std::vector<BisectionID> &bisection_ids) const;
// find levels that offer good distribution of average cell sizes
void SearchLevels(const std::vector<SizedID> &implicit_tree,
const BisectionGraph &graph,
const std::vector<BisectionID> &bisection_ids) const;
// set cell_ids[i] == INFTY to exclude element
LevelMetrics AnalyseLevel(const BisectionGraph &graph,
const std::vector<std::uint32_t> &cell_ids) const;
std::vector<std::uint32_t>
ComputeCellIDs(std::vector<std::pair<BisectionID, std::int32_t>> &prefixes,
const BisectionGraph &graph,
const std::vector<BisectionID> &bisection_ids) const;
};
} // namespace partition
} // namespace osrm
#endif // OSRM_PARTITION_ANNOTATE_HPP_

View File

@ -35,7 +35,7 @@ class InertialFlow
SpatialOrder MakeSpatialOrder(double ratio, double slope) const;
// Makes n cuts with different spatial orders and returns the best.
DinicMaxFlow::MinCut BestMinCut(std::size_t n, double ratio) const;
DinicMaxFlow::MinCut BestMinCut(std::size_t n, double ratio, double balance) const;
// The subgraph to partition into two parts.
const GraphView &view;

View File

@ -4,6 +4,7 @@
#include "partition/bisection_graph.hpp"
#include "partition/graph_view.hpp"
#include "partition/recursive_bisection_state.hpp"
#include "util/typedefs.hpp"
#include <cstddef>
#include <vector>
@ -23,7 +24,7 @@ class RecursiveBisection
const std::size_t num_optimizing_cuts,
const std::size_t small_component_size);
const std::vector<RecursiveBisectionState::BisectionID> &BisectionIDs() const;
const std::vector<BisectionID> &BisectionIDs() const;
private:
BisectionGraph &bisection_graph;

View File

@ -20,7 +20,6 @@ class RecursiveBisectionState
{
public:
// The ID in the partition array
using BisectionID = std::uint32_t;
using NodeIterator = BisectionGraph::ConstNodeIterator;
RecursiveBisectionState(BisectionGraph &bisection_graph);
@ -42,7 +41,11 @@ class RecursiveBisectionState
const std::vector<BisectionID> &BisectionIDs() const;
// return the depth encoded in the SCCs
std::uint32_t SCCDepth() const;
private:
std::uint32_t scc_levels;
BisectionGraph &bisection_graph;
std::vector<BisectionID> bisection_ids;
};

View File

@ -1,19 +0,0 @@
#ifndef OSRM_PARTITION_RECURSIVE_BISECTION_STATS_HPP_
#define OSRM_PARTITION_RECURSIVE_BISECTION_STATS_HPP_
#include "partition/bisection_graph.hpp"
#include "partition/recursive_bisection_state.hpp"
#include <vector>
namespace osrm
{
namespace partition
{
// generates some statistics on a recursive bisection to describe its quality/parameters
void printBisectionStats(std::vector<RecursiveBisectionState::BisectionID> const &bisection_ids,
const BisectionGraph &graph);
} // namespace partition
} // namespace osrm
#endif // OSRM_PARTITION_RECURSIVE_BISECTION_STATS_HPP_

View File

@ -81,26 +81,21 @@ template <typename EdgeDataT> struct SortableEdgeWithData : SortableEdgeWithData
} // namespace static_graph_details
template <typename NodeT, typename EdgeT, bool UseSharedMemory = false> class FlexibleStaticGraph
template <typename EdgeDataT, bool UseSharedMemory = false> class StaticGraph
{
static_assert(traits::HasFirstEdgeMember<NodeT>::value,
"Model for compatible Node type requires .first_edge member attribute");
static_assert(traits::HasTargetMember<EdgeT>::value,
"Model for compatible Node type requires .target member attribute");
public:
using NodeIterator = static_graph_details::NodeIterator;
using EdgeIterator = static_graph_details::EdgeIterator;
using EdgeRange = range<EdgeIterator>;
using NodeArrayEntry = NodeT;
using EdgeArrayEntry = EdgeT;
using NodeArrayEntry = static_graph_details::NodeArrayEntry;
using EdgeArrayEntry = static_graph_details::EdgeArrayEntry<EdgeDataT>;
EdgeRange GetAdjacentEdgeRange(const NodeID node) const
{
return irange(BeginEdges(node), EndEdges(node));
}
template <typename ContainerT> FlexibleStaticGraph(const int nodes, const ContainerT &graph)
template <typename ContainerT> StaticGraph(const int nodes, const ContainerT &graph)
{
BOOST_ASSERT(std::is_sorted(const_cast<ContainerT &>(graph).begin(),
const_cast<ContainerT &>(graph).end()));
@ -128,14 +123,15 @@ template <typename NodeT, typename EdgeT, bool UseSharedMemory = false> class Fl
for (const auto i : irange(node_array[node].first_edge, e))
{
edge_array[i].target = graph[edge].target;
CopyDataIfAvailable(edge_array[i], graph[edge], traits::HasDataMember<EdgeT>{});
CopyDataIfAvailable(
edge_array[i], graph[edge], traits::HasDataMember<EdgeArrayEntry>{});
edge++;
}
}
}
FlexibleStaticGraph(typename ShM<NodeT, UseSharedMemory>::vector &nodes,
typename ShM<EdgeT, UseSharedMemory>::vector &edges)
StaticGraph(typename ShM<NodeArrayEntry, UseSharedMemory>::vector &nodes,
typename ShM<EdgeArrayEntry, UseSharedMemory>::vector &edges)
{
number_of_nodes = static_cast<decltype(number_of_nodes)>(nodes.size() - 1);
number_of_edges = static_cast<decltype(number_of_edges)>(edges.size());
@ -198,7 +194,7 @@ template <typename NodeT, typename EdgeT, bool UseSharedMemory = false> class Fl
EdgeIterator
FindSmallestEdge(const NodeIterator from, const NodeIterator to, FilterFunction &&filter) const
{
static_assert(traits::HasDataMember<EdgeT>::value,
static_assert(traits::HasDataMember<EdgeArrayEntry>::value,
"Filtering on .data not possible without .data member attribute");
EdgeIterator smallest_edge = SPECIAL_EDGEID;
@ -243,13 +239,13 @@ template <typename NodeT, typename EdgeT, bool UseSharedMemory = false> class Fl
private:
template <typename OtherEdge>
void CopyDataIfAvailable(EdgeT &into, const OtherEdge &from, std::true_type)
void CopyDataIfAvailable(EdgeArrayEntry &into, const OtherEdge &from, std::true_type)
{
into.data = from.data;
}
template <typename OtherEdge>
void CopyDataIfAvailable(EdgeT &into, const OtherEdge &from, std::false_type)
void CopyDataIfAvailable(EdgeArrayEntry &into, const OtherEdge &from, std::false_type)
{
// Graph has no .data member, never copy even if `from` has a .data member.
(void)into;
@ -259,15 +255,11 @@ template <typename NodeT, typename EdgeT, bool UseSharedMemory = false> class Fl
NodeIterator number_of_nodes;
EdgeIterator number_of_edges;
typename ShM<NodeT, UseSharedMemory>::vector node_array;
typename ShM<EdgeT, UseSharedMemory>::vector edge_array;
typename ShM<NodeArrayEntry, UseSharedMemory>::vector node_array;
typename ShM<EdgeArrayEntry, UseSharedMemory>::vector edge_array;
};
template <typename EdgeDataT, bool UseSharedMemory = false>
using StaticGraph = FlexibleStaticGraph<static_graph_details::NodeArrayEntry,
static_graph_details::EdgeArrayEntry<EdgeDataT>,
UseSharedMemory>;
}
}
} // namespace util
} // namespace osrm
#endif // STATIC_GRAPH_HPP

View File

@ -61,6 +61,7 @@ using EdgeWeight = std::int32_t;
using TurnPenalty = std::int16_t; // turn penalty in 100ms units
static const std::size_t INVALID_INDEX = std::numeric_limits<std::size_t>::max();
using BisectionID = std::uint32_t;
using LaneID = std::uint8_t;
static const LaneID INVALID_LANEID = std::numeric_limits<LaneID>::max();

View File

@ -0,0 +1,387 @@
#include "partition/annotated_partition.hpp"
#include <algorithm>
#include <climits> // for CHAR_BIT
#include <cstddef>
#include <cstdint>
#include <iostream>
#include <limits>
#include <map>
#include <numeric>
#include <queue>
#include <string>
#include <unordered_map>
#include "util/timing_util.hpp"
namespace osrm
{
namespace partition
{
namespace
{
// the shift value needed to access the most significant bit of the bisection ID
const constexpr auto SHIFT_TO_MSB_BISECTION_ID = sizeof(BisectionID) * CHAR_BIT - 1;
// an invalid ID for a cell
const constexpr std::uint32_t INVALID_CELLID = std::numeric_limits<std::uint32_t>::max();
auto masked(const BisectionID id, const std::int32_t level)
{
// special treatment for negative level
if (level == -1)
return 0u;
// 0.01.1 with 1 starting at the level+1_th most significant bit (level = 0 -> 01..1)
const auto cut_below_level = (1 << (SHIFT_TO_MSB_BISECTION_ID - level)) - 1;
const auto mask = std::numeric_limits<BisectionID>::max() ^ cut_below_level;
return id & mask;
}
// create a comparator for a given level
auto makeCompare(const std::uint32_t level)
{
return [level](const AnnotatedPartition::SizedID lhs, const AnnotatedPartition::SizedID rhs) {
return masked(lhs.id, level) < masked(rhs.id, level);
};
}
// build a tree of cells from the IDs present:
auto leftChild(const BisectionID id_prefix, const std::int32_t /*level*/) { return id_prefix; }
// given the prefix 10.... on level 1 (second level), the the right child would be
// 101.... on level 2
auto rightChild(const BisectionID id_prefix, const std::int32_t level)
{
return id_prefix | (1 << (SHIFT_TO_MSB_BISECTION_ID - (level + 1)));
}
// get the range of all children
auto getChildrenRange(const std::vector<AnnotatedPartition::SizedID> &implicit_tree,
const BisectionID id_prefix,
const std::int32_t level)
{
AnnotatedPartition::SizedID id = {id_prefix, 0};
// find all elements of the same prefix as id_prefi
auto range =
std::equal_range(implicit_tree.begin(), implicit_tree.end(), id, makeCompare(level));
// don't ever return our sentinel element as included
if (range.second == implicit_tree.end())
--range.second;
return range;
};
auto getCellSize(const std::vector<AnnotatedPartition::SizedID> &implicit_tree,
const BisectionID id_prefix,
const std::uint32_t level)
{
auto range = getChildrenRange(implicit_tree, id_prefix, level);
return range.second->count - range.first->count;
}
bool hasChildren(const std::vector<AnnotatedPartition::SizedID> &implicit_tree,
const BisectionID id_prefix,
const std::uint32_t level)
{
auto range = getChildrenRange(implicit_tree, id_prefix, level);
return std::distance(range.first, range.second) > 1;
}
} // namespace
AnnotatedPartition::AnnotatedPartition(const BisectionGraph &graph,
const std::vector<BisectionID> &bisection_ids)
{
// create a sorted vector of bisection ids that exist in the network
std::vector<SizedID> implicit_tree = [&]() {
std::map<BisectionID, SizedID> existing_ids;
// insert an ID into the sized_id set or increase the count if the element should be already
// present in the set of known ids
const auto insert_or_augment = [&existing_ids](const BisectionID id) {
SizedID sized_id = {id, 1};
auto maybe_existing_id = existing_ids.find(id);
if (maybe_existing_id == existing_ids.end())
existing_ids[id] = sized_id;
else
maybe_existing_id->second.count++;
};
std::for_each(bisection_ids.begin(), bisection_ids.end(), insert_or_augment);
std::vector<SizedID> result;
result.resize(existing_ids.size() + 1);
std::transform(existing_ids.begin(),
existing_ids.end(),
result.begin(),
[](const auto &pair) { return pair.second; });
// sentinel
result.back() = {std::numeric_limits<BisectionID>::max(), 0};
return result;
}();
// calculate a prefix sum over all sorted IDs, this allows to get the size of any partition in
// the array/level based on the prefix and lower bound on prefixes.
// e.g 00,01,10,11 allow to search for (0) (1) to find (00) and (10) as lower bounds. The
// difference in count is the size of all cells in the left part of the partition.
std::transform(implicit_tree.begin(),
implicit_tree.end(),
implicit_tree.begin(),
[sum = std::size_t{0}](SizedID id) mutable {
const auto new_sum = sum + id.count;
id.count = sum;
sum = new_sum;
return id;
});
PrintBisection(implicit_tree, graph, bisection_ids);
SearchLevels(implicit_tree, graph, bisection_ids);
}
void AnnotatedPartition::PrintBisection(const std::vector<SizedID> &implicit_tree,
const BisectionGraph &graph,
const std::vector<BisectionID> &bisection_ids) const
{
// print some statistics on the bisection tree
std::queue<BisectionID> id_queue;
id_queue.push(0);
const auto add_child = [&id_queue, &implicit_tree](const BisectionID prefix,
const std::uint32_t level) {
const auto child_range = getChildrenRange(implicit_tree, prefix, level);
if (std::distance(child_range.first, child_range.second) > 1)
id_queue.push(prefix);
};
std::vector<std::pair<BisectionID, std::int32_t>> current_level;
for (std::int32_t level = -1; !id_queue.empty(); ++level)
{
auto level_size = id_queue.size();
current_level.clear();
while (level_size--)
{
const auto prefix = id_queue.front();
id_queue.pop();
if (level == -1 || hasChildren(implicit_tree, prefix, level))
{
current_level.push_back(
std::pair<BisectionID, std::uint32_t>(leftChild(prefix, level), level + 1));
current_level.push_back(
std::pair<BisectionID, std::uint32_t>(rightChild(prefix, level), level + 1));
}
add_child(leftChild(prefix, level), level);
add_child(rightChild(prefix, level), level);
}
if (!current_level.empty())
{
const auto cell_ids = ComputeCellIDs(current_level, graph, bisection_ids);
const auto stats = AnalyseLevel(graph, cell_ids);
stats.logMachinereadable(std::cout, "bisection", level, level == -1);
}
}
}
void AnnotatedPartition::SearchLevels(const std::vector<SizedID> &implicit_tree,
const BisectionGraph &graph,
const std::vector<BisectionID> &bisection_ids) const
{
std::vector<std::pair<BisectionID, std::int32_t>> current_level;
// start searching with level 0 at prefix 0
current_level.push_back({static_cast<BisectionID>(0), -1});
std::int32_t level = -1;
const auto print_level = [&]() {
if (current_level.empty())
return;
const auto cell_ids = ComputeCellIDs(current_level, graph, bisection_ids);
const auto stats = AnalyseLevel(graph, cell_ids);
stats.logMachinereadable(std::cout, "dfs-balanced", level, level == -1);
++level;
};
std::size_t max_size = 0.5 * graph.NumberOfNodes();
std::queue<std::pair<BisectionID, std::int32_t>> id_queue;
while (!current_level.empty())
{
std::size_t total_size = 0;
std::size_t count = 0;
for (auto element : current_level)
{
// don't relax final cells
if (element.second == -1 || hasChildren(implicit_tree, element.first, element.second))
{
total_size += getCellSize(
implicit_tree, leftChild(element.first, element.second), element.second + 1);
id_queue.push(std::pair<BisectionID, std::uint32_t>(
leftChild(element.first, element.second), element.second + 1));
total_size += getCellSize(
implicit_tree, rightChild(element.first, element.second), element.second + 1);
id_queue.push(std::pair<BisectionID, std::uint32_t>(
rightChild(element.first, element.second), element.second + 1));
count += 2;
}
}
auto avg_size = (total_size / static_cast<double>(count));
current_level.clear();
const auto relax = [&id_queue, &implicit_tree, avg_size, &current_level](
const std::pair<BisectionID, std::uint32_t> &element) {
const auto size = getCellSize(implicit_tree, element.first, element.second);
if (!hasChildren(implicit_tree, element.first, element.second))
{
current_level.push_back(element);
}
else
{
const auto left = leftChild(element.first, element.second);
const auto right = rightChild(element.first, element.second);
const auto get_penalty = [avg_size](const auto size) {
return std::abs(size - avg_size);
};
if (get_penalty(size) <
0.5 * (get_penalty(getCellSize(implicit_tree, left, element.second + 1)) +
get_penalty(getCellSize(implicit_tree, right, element.second + 1))))
{
current_level.push_back(element);
}
else
{
id_queue.push(std::pair<BisectionID, std::uint32_t>(left, element.second + 1));
id_queue.push(std::pair<BisectionID, std::uint32_t>(right, element.second + 1));
}
}
};
while (!id_queue.empty())
{
relax(id_queue.front());
id_queue.pop();
}
print_level();
max_size *= 0.5;
}
}
AnnotatedPartition::LevelMetrics
AnnotatedPartition::AnalyseLevel(const BisectionGraph &graph,
const std::vector<std::uint32_t> &cell_ids) const
{
std::unordered_map<std::uint32_t, std::size_t> cell_sizes;
std::unordered_map<std::uint32_t, std::size_t> border_nodes;
std::unordered_map<std::uint32_t, std::size_t> border_arcs;
// compute basic metrics of the level
std::size_t border_nodes_total = 0;
std::size_t border_arcs_total = 0;
std::size_t contained_nodes = 0;
// only border nodes on the lowest level can be border nodes in general
for (const auto &node : graph.Nodes())
{
const auto cell_id = cell_ids[node.original_id];
if (cell_id == INVALID_CELLID)
continue;
++contained_nodes;
const auto edge_range = graph.Edges(node);
const auto border_arcs_at_node = std::count_if(
edge_range.begin(), edge_range.end(), [&cell_id, &cell_ids, &graph](const auto &edge) {
const auto target_cell_id = cell_ids[graph.Node(edge.target).original_id];
return target_cell_id != cell_id;
});
cell_sizes[cell_id]++;
border_arcs[cell_id] += border_arcs_at_node;
border_arcs_total += border_arcs_at_node;
if (border_arcs_at_node)
{
border_nodes[cell_id]++;
++border_nodes_total;
}
}
const auto by_size = [](const std::pair<std::uint32_t, std::size_t> &lhs,
const std::pair<std::uint32_t, std::size_t> &rhs) {
return lhs.second < rhs.second;
};
const auto max_nodes =
border_nodes.empty()
? 0
: std::max_element(border_nodes.begin(), border_nodes.end(), by_size)->second;
const auto max_arcs =
border_arcs.empty()
? 0
: std::max_element(border_arcs.begin(), border_arcs.end(), by_size)->second;
const auto squarded_size = [](const std::size_t accumulated,
const std::pair<std::uint32_t, std::size_t> &element) {
return accumulated + element.second * element.second;
};
const auto memory =
4 * std::accumulate(border_arcs.begin(), border_nodes.end(), std::size_t(0), squarded_size);
std::vector<std::size_t> cell_sizes_vec;
cell_sizes_vec.resize(cell_sizes.size());
std::transform(cell_sizes.begin(),
cell_sizes.end(),
cell_sizes_vec.begin(),
[](const auto &pair) { return pair.second; });
return {border_nodes_total,
border_arcs_total,
contained_nodes,
border_nodes.size(),
max_nodes,
max_arcs,
memory,
std::move(cell_sizes_vec)};
}
std::vector<std::uint32_t>
AnnotatedPartition::ComputeCellIDs(std::vector<std::pair<BisectionID, std::int32_t>> &prefixes,
const BisectionGraph &graph,
const std::vector<BisectionID> &bisection_ids) const
{
std::vector<std::uint32_t> cell_ids(graph.NumberOfNodes(), INVALID_CELLID);
std::sort(prefixes.begin(), prefixes.end(), [](const auto lhs, const auto rhs) {
return lhs.first < rhs.first;
});
for (const auto &node : graph.Nodes())
{
// find the cell_id of node in the current levels
const auto id = bisection_ids[node.original_id];
const auto is_prefixed_by = [id](const auto &prefix) {
return masked(id, prefix.second) == prefix.first;
};
const auto prefix = std::lower_bound(
prefixes.begin(), prefixes.end(), id, [&](const auto prefix, const BisectionID id) {
return prefix.first < masked(id, prefix.second);
});
if (prefix == prefixes.end())
continue;
if (is_prefixed_by(*prefix))
cell_ids[node.original_id] = std::distance(prefixes.begin(), prefix);
}
return cell_ids;
}
} // namespace partition
} // namespace osrm

View File

@ -25,9 +25,7 @@ DinicMaxFlow::MinCut InertialFlow::ComputePartition(const std::size_t num_slopes
const double balance,
const double source_sink_rate)
{
auto cut = BestMinCut(num_slopes, source_sink_rate);
return cut;
return BestMinCut(num_slopes, source_sink_rate, balance);
}
InertialFlow::SpatialOrder InertialFlow::MakeSpatialOrder(const double ratio,
@ -86,23 +84,36 @@ InertialFlow::SpatialOrder InertialFlow::MakeSpatialOrder(const double ratio,
return order;
}
DinicMaxFlow::MinCut InertialFlow::BestMinCut(const std::size_t n, const double ratio) const
DinicMaxFlow::MinCut
InertialFlow::BestMinCut(const std::size_t n, const double ratio, const double balance) const
{
DinicMaxFlow::MinCut best;
best.num_edges = -1;
const auto get_balance = [this](const auto num_nodes_source) {
double ratio = static_cast<double>(view.NumberOfNodes() - num_nodes_source) /
static_cast<double>(num_nodes_source);
return std::abs(ratio - 1.0);
const auto get_balance = [this, balance](const auto num_nodes_source) {
const auto perfect_balance = view.NumberOfNodes() / 2;
const auto allowed_balance = balance * perfect_balance;
const auto bigger_side =
std::max(num_nodes_source, view.NumberOfNodes() - num_nodes_source);
if (bigger_side > allowed_balance)
return bigger_side / static_cast<double>(allowed_balance);
else
return 1.0;
};
auto best_balance = 10000; // get_balance(best.num_nodes_source);
auto best_balance = 1;
std::mutex lock;
tbb::blocked_range<std::size_t> range{0, n, 1};
const auto balance_delta = [this](const auto num_nodes_source) {
const std::int64_t difference =
static_cast<std::int64_t>(view.NumberOfNodes()) / 2 - num_nodes_source;
return std::abs(difference);
};
tbb::parallel_for(range, [&, this](const auto &chunk) {
for (auto round = chunk.begin(), end = chunk.end(); round != end; ++round)
{
@ -116,7 +127,9 @@ DinicMaxFlow::MinCut InertialFlow::BestMinCut(const std::size_t n, const double
std::lock_guard<std::mutex> guard{lock};
// Swap to keep the destruction of the old object outside of critical section.
if (std::tie(cut.num_edges, cut_balance) < std::tie(best.num_edges, best_balance))
if (cut.num_edges * cut_balance < best.num_edges * best_balance ||
(cut.num_edges == best.num_edges &&
balance_delta(cut.num_nodes_source) < balance_delta(best.num_nodes_source)))
{
best_balance = cut_balance;
std::swap(best, cut);

View File

@ -1,7 +1,7 @@
#include "partition/partitioner.hpp"
#include "partition/annotated_partition.hpp"
#include "partition/bisection_graph.hpp"
#include "partition/recursive_bisection.hpp"
#include "partition/recursive_bisection_stats.hpp"
#include "storage/io.hpp"
#include "util/coordinate.hpp"
@ -19,6 +19,7 @@
#include "util/geojson_debug_logger.hpp"
#include "util/geojson_debug_policies.hpp"
#include "util/json_container.hpp"
#include "util/timing_util.hpp"
namespace osrm
{
@ -65,6 +66,27 @@ CompressedNodeBasedGraph LoadCompressedNodeBasedGraph(const std::string &path)
return graph;
}
void LogStatistics(const std::string &filename, std::vector<std::uint32_t> bisection_ids)
{
auto compressed_node_based_graph = LoadCompressedNodeBasedGraph(filename);
util::Log() << "Loaded compressed node based graph: "
<< compressed_node_based_graph.edges.size() << " edges, "
<< compressed_node_based_graph.coordinates.size() << " nodes";
groupEdgesBySource(begin(compressed_node_based_graph.edges),
end(compressed_node_based_graph.edges));
auto graph =
makeBisectionGraph(compressed_node_based_graph.coordinates,
adaptToBisectionEdge(std::move(compressed_node_based_graph.edges)));
TIMER_START(annotation);
AnnotatedPartition partition(graph, bisection_ids);
TIMER_STOP(annotation);
std::cout << "Annotation took " << TIMER_SEC(annotation) << " seconds" << std::endl;
}
void LogGeojson(const std::string &filename, std::vector<std::uint32_t> bisection_ids)
{
// reload graph, since we destroyed the old one
@ -87,18 +109,6 @@ void LogGeojson(const std::string &filename, std::vector<std::uint32_t> bisectio
return level;
};
const auto reverse_bits = [](std::uint32_t x) {
x = ((x >> 1) & 0x55555555u) | ((x & 0x55555555u) << 1);
x = ((x >> 2) & 0x33333333u) | ((x & 0x33333333u) << 2);
x = ((x >> 4) & 0x0f0f0f0fu) | ((x & 0x0f0f0f0fu) << 4);
x = ((x >> 8) & 0x00ff00ffu) | ((x & 0x00ff00ffu) << 8);
x = ((x >> 16) & 0xffffu) | ((x & 0xffffu) << 16);
return x;
};
std::transform(bisection_ids.begin(), bisection_ids.end(), bisection_ids.begin(), reverse_bits);
printBisectionStats(bisection_ids, graph);
std::vector<std::vector<util::Coordinate>> border_vertices(33);
for (NodeID nid = 0; nid < graph.NumberOfNodes(); ++nid)
@ -151,6 +161,10 @@ int Partitioner::Run(const PartitionConfig &config)
makeBisectionGraph(compressed_node_based_graph.coordinates,
adaptToBisectionEdge(std::move(compressed_node_based_graph.edges)));
util::Log() << " running partition: " << config.maximum_cell_size << " " << config.balance
<< " " << config.boundary_factor << " " << config.num_optimizing_cuts << " "
<< config.small_component_size
<< " # max_cell_size balance boundary cuts small_component_size";
RecursiveBisection recursive_bisection(graph,
config.maximum_cell_size,
config.balance,
@ -158,7 +172,7 @@ int Partitioner::Run(const PartitionConfig &config)
config.num_optimizing_cuts,
config.small_component_size);
LogGeojson(config.compressed_node_based_graph_path.string(),
LogStatistics(config.compressed_node_based_graph_path.string(),
recursive_bisection.BisectionIDs());
return 0;

View File

@ -60,8 +60,8 @@ RecursiveBisection::RecursiveBisection(BisectionGraph &bisection_graph_,
std::vector<TreeNode> forest;
forest.reserve(last - first);
std::transform(first, last, std::back_inserter(forest), [](auto graph) {
return TreeNode{std::move(graph), 0};
std::transform(first, last, std::back_inserter(forest), [this](auto graph) {
return TreeNode{std::move(graph), internal_state.SCCDepth()};
});
using Feeder = tbb::parallel_do_feeder<TreeNode>;
@ -76,7 +76,7 @@ RecursiveBisection::RecursiveBisection(BisectionGraph &bisection_graph_,
node.graph.Begin(), node.graph.End(), node.depth, partition.flags);
const auto terminal = [&](const auto &node) {
const auto maximum_depth = sizeof(RecursiveBisectionState::BisectionID) * CHAR_BIT;
const auto maximum_depth = sizeof(BisectionID) * CHAR_BIT;
const auto too_small = node.graph.NumberOfNodes() < maximum_cell_size;
const auto too_deep = node.depth >= maximum_depth;
return too_small || too_deep;
@ -100,7 +100,7 @@ RecursiveBisection::RecursiveBisection(BisectionGraph &bisection_graph_,
util::Log() << "Full bisection done in " << TIMER_SEC(bisection) << "s";
}
const std::vector<RecursiveBisectionState::BisectionID> &RecursiveBisection::BisectionIDs() const
const std::vector<BisectionID> &RecursiveBisection::BisectionIDs() const
{
return internal_state.BisectionIDs();
}

View File

@ -3,11 +3,9 @@
#include "partition/tarjan_graph_wrapper.hpp"
#include <algorithm>
#include <climits> // for CHAR_BIT
#include <numeric>
// TODO remove
#include <bitset>
#include <iostream>
#include <set>
#include <unordered_map>
namespace osrm
@ -16,15 +14,14 @@ namespace partition
{
RecursiveBisectionState::RecursiveBisectionState(BisectionGraph &bisection_graph_)
: bisection_graph(bisection_graph_)
: scc_levels(0), bisection_graph(bisection_graph_)
{
bisection_ids.resize(bisection_graph.NumberOfNodes(), BisectionID{0});
}
RecursiveBisectionState::~RecursiveBisectionState() {}
RecursiveBisectionState::BisectionID
RecursiveBisectionState::GetBisectionID(const NodeID node) const
BisectionID RecursiveBisectionState::GetBisectionID(const NodeID node) const
{
return bisection_ids[node];
}
@ -35,12 +32,13 @@ RecursiveBisectionState::ApplyBisection(const NodeIterator const_begin,
const std::size_t depth,
const std::vector<bool> &partition)
{
BOOST_ASSERT(depth >= scc_levels);
// ensure that the iterators belong to the graph
BOOST_ASSERT(bisection_graph.GetID(*const_begin) < bisection_graph.NumberOfNodes() &&
bisection_graph.GetID(*const_begin) + std::distance(const_begin, const_end) <=
bisection_graph.NumberOfNodes());
// augment the partition ids
const auto flag = BisectionID{1} << depth;
const auto flag = BisectionID{1} << (sizeof(BisectionID) * CHAR_BIT - depth - 1);
for (auto itr = const_begin; itr != const_end; ++itr)
{
const auto nid = std::distance(const_begin, itr);
@ -129,9 +127,11 @@ RecursiveBisectionState::PrePartitionWithSCC(const std::size_t small_component_s
std::vector<GraphView> views;
auto last = bisection_graph.CBegin();
auto last_id = transform_id(bisection_graph.Begin()->original_id);
std::set<std::size_t> ordered_component_ids;
for (auto itr = bisection_graph.CBegin(); itr != bisection_graph.CEnd(); ++itr)
{
auto itr_id = transform_id(itr->original_id);
ordered_component_ids.insert(itr_id);
if (last_id != itr_id)
{
views.push_back(GraphView(bisection_graph, last, itr));
@ -151,14 +151,34 @@ RecursiveBisectionState::PrePartitionWithSCC(const std::size_t small_component_s
if (!has_small_component)
views.push_back(GraphView(bisection_graph, bisection_graph.CEnd(), bisection_graph.CEnd()));
// apply scc as bisections, we need scc_level bits for this with scc_levels =
// ceil(log_2(components))
scc_levels = ceil(log(views.size()) / log(2.0));
const auto conscutive_component_id = [&](const NodeID nid) {
const auto component_id = transform_id(nid);
const auto itr = ordered_component_ids.find(component_id);
BOOST_ASSERT(itr != ordered_component_ids.end());
BOOST_ASSERT(static_cast<std::size_t>(std::distance(ordered_component_ids.begin(), itr)) <
ordered_component_ids.size());
return std::distance(ordered_component_ids.begin(), itr);
};
const auto shift = sizeof(BisectionID) * CHAR_BIT - scc_levels;
// store the component ids as first part of the bisection id
for (const auto &node : bisection_graph.Nodes())
bisection_ids[node.original_id] = conscutive_component_id(node.original_id) << shift;
return views;
}
const std::vector<RecursiveBisectionState::BisectionID> &
RecursiveBisectionState::BisectionIDs() const
const std::vector<BisectionID> &RecursiveBisectionState::BisectionIDs() const
{
return bisection_ids;
}
std::uint32_t RecursiveBisectionState::SCCDepth() const { return scc_levels; }
} // namespace partition
} // namespace osrm

View File

@ -1,113 +0,0 @@
#include "partition/recursive_bisection_stats.hpp"
#include <boost/assert.hpp>
#include <bitset>
#include <cstddef>
#include <iostream>
#include <unordered_map>
#include <unordered_set>
namespace osrm
{
namespace partition
{
void printBisectionStats(std::vector<RecursiveBisectionState::BisectionID> const &bisection_ids,
const BisectionGraph &graph)
{
BOOST_ASSERT(graph.NumberOfNodes() == bisection_ids.size());
std::size_t total_border_nodes = 0;
std::unordered_map<RecursiveBisectionState::BisectionID, std::size_t> cell_sizes[32];
std::unordered_map<RecursiveBisectionState::BisectionID, std::size_t> border_nodes[32];
std::unordered_set<RecursiveBisectionState::BisectionID> all_ids[32];
std::uint32_t flag = 0;
for (std::uint32_t level = 0; level < 32; ++level)
{
std::uint32_t bit = 1u << (31 - level);
flag |= bit;
for (std::size_t i = 0; i < bisection_ids.size(); ++i)
all_ids[level].insert(bisection_ids[i] & flag);
if (level > 0 && all_ids[level - 1] == all_ids[level])
break;
std::cout << "Level" << std::endl;
for (auto itr : all_ids[level])
std::cout << "\t" << std::bitset<32>(itr) << std::endl;
for (const auto &node : graph.Nodes())
{
const auto bisection_id_node = bisection_ids[node.original_id];
// subrange is not partitioned anymore
if (all_ids[level].count((bisection_id_node & flag) ^ bit) == 0)
continue;
auto is_border_node = false;
for (const auto &edge : graph.Edges(node))
{
if (bisection_ids[edge.target] != bisection_id_node)
is_border_node = true;
}
if (is_border_node && level == 0)
++total_border_nodes;
cell_sizes[level][bisection_id_node & flag]++;
if (is_border_node)
{
for (const auto &edge : graph.Edges(node))
{
if ((bisection_id_node & flag) != (bisection_ids[edge.target] & flag))
{
border_nodes[level][bisection_id_node & flag]++;
break;
}
}
}
}
}
std::cout << "Partition statistics\n";
std::cout << "Total border vertices: " << total_border_nodes << std::endl;
unsigned level = 0;
do
{
std::size_t min_size = -1, max_size = 0, total_size = 0;
std::size_t min_border = -1, max_border = 1, total_border = 0;
const auto summarize =
[](const std::unordered_map<RecursiveBisectionState::BisectionID, std::size_t> &map,
std::size_t &min,
std::size_t &max,
std::size_t &total) {
for (const auto itr : map)
{
min = std::min(min, itr.second);
max = std::max(max, itr.second);
total += itr.second;
}
};
summarize(cell_sizes[level], min_size, max_size, total_size);
summarize(border_nodes[level], min_border, max_border, total_border);
std::cout << "Level: " << level << " Cells: " << cell_sizes[level].size();
if (cell_sizes[level].size() > 1)
std::cout << " Border: " << min_border << " " << max_border << " "
<< total_border / (double)cell_sizes[level].size();
std::cout << " Cell Sizes: " << min_size << " " << max_size << " "
<< total_size / (double)cell_sizes[level].size()
<< " Total Vertices In Level: " << total_size;
std::cout << std::endl;
} while (level < 31 && cell_sizes[++level].size() > 1);
}
} // namespace partition
} // namespace osrm

View File

@ -3,6 +3,7 @@
#include "partition/recursive_bisection_state.hpp"
#include <algorithm>
#include <climits>
#include <vector>
#include <boost/test/test_case_template.hpp>
@ -51,7 +52,8 @@ BOOST_AUTO_TEST_CASE(separate_top_bottom)
BOOST_CHECK_EQUAL(right.NumberOfNodes(), 4);
for (const auto &node : right.Nodes())
{
BOOST_CHECK_EQUAL(bisection_state.GetBisectionID(node.original_id), 1);
BOOST_CHECK_EQUAL(bisection_state.GetBisectionID(node.original_id),
1 << (sizeof(BisectionID) * CHAR_BIT - 1));
auto id = right.GetID(node);
const auto compare = makeCoordinate(id, 1, step_size);
BOOST_CHECK_EQUAL(compare, node.coordinate);
@ -102,7 +104,8 @@ BOOST_AUTO_TEST_CASE(separate_top_bottom_copy)
for (NodeID id = 0; id < right.NumberOfNodes(); ++id)
{
const auto &node = right.Node(id);
BOOST_CHECK_EQUAL(bisection_state.GetBisectionID(node.original_id), 1);
BOOST_CHECK_EQUAL(bisection_state.GetBisectionID(node.original_id),
1 << (sizeof(BisectionID) * CHAR_BIT - 1));
const auto compare = makeCoordinate(id, 1, step_size);
BOOST_CHECK_EQUAL(compare, node.coordinate);
BOOST_CHECK(id < right.NumberOfNodes());
@ -162,7 +165,8 @@ BOOST_AUTO_TEST_CASE(separate_left_right)
auto right_compare = right_coordinates.begin();
for (const auto &node : right.Nodes())
{
BOOST_CHECK_EQUAL(bisection_state.GetBisectionID(node.original_id), 1);
BOOST_CHECK_EQUAL(bisection_state.GetBisectionID(node.original_id),
1 << (sizeof(BisectionID) * CHAR_BIT - 1));
auto id = right.GetID(node);
const auto compare = *right_compare++;
BOOST_CHECK_EQUAL(compare, node.coordinate);