Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Refactor R-Tree updates and edges loading

Browse Source
This commit is contained in:
Michael Krasnyk 2017-01-10 09:48:02 +01:00 committed by Patrick Niklaus
parent 29f736f1c8
commit 71e7d6d6b8
1 changed files with 91 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

217
src/contractor/contractor.cpp
View File

@ -85,9 +85,9 @@ struct Turn final
: from(from), via(via), to(to) : from(from), via(via), to(to)
{ {
} }
Turn(const OSMNodeID from, const OSMNodeID via, const OSMNodeID to) Turn(const osrm::extractor::lookup::PenaltyBlock &turn)
: from(static_cast<std::uint64_t>(from)), via(static_cast<std::uint64_t>(via)), : from(static_cast<std::uint64_t>(turn.from_id)),
to(static_cast<std::uint64_t>(to)) via(static_cast<std::uint64_t>(turn.via_id)), to(static_cast<std::uint64_t>(turn.to_id))
{ {
} }
bool operator<(const Turn &rhs) const bool operator<(const Turn &rhs) const
@ -139,8 +139,8 @@ template <typename Key, typename Value> struct CSVFilesParser
using KeyRule = qi::rule<Iterator, Key()>; using KeyRule = qi::rule<Iterator, Key()>;
using ValueRule = qi::rule<Iterator, Value()>; using ValueRule = qi::rule<Iterator, Value()>;
CSVFilesParser(const KeyRule &key_rule, const ValueRule &value_rule) CSVFilesParser(std::size_t start_index, const KeyRule &key_rule, const ValueRule &value_rule)
: key_rule(key_rule), value_rule(value_rule) : start_index(start_index), key_rule(key_rule), value_rule(value_rule)
{ {
} }
@ -154,8 +154,7 @@ template <typename Key, typename Value> struct CSVFilesParser
tbb::parallel_for(std::size_t{0}, tbb::parallel_for(std::size_t{0},
csv_filenames.size(), csv_filenames.size(),
[&](const std::size_t idx) { [&](const std::size_t idx) {
// id starts at one, 0 means we assigned the weight from profile auto local = ParseCSVFile(csv_filenames[idx], start_index + idx);
auto local = ParseCSVFile(csv_filenames[idx], idx + 1);
{ // Merge local CSV results into a flat global vector { // Merge local CSV results into a flat global vector
tbb::spin_mutex::scoped_lock _{mutex}; tbb::spin_mutex::scoped_lock _{mutex};
@ -209,6 +208,7 @@ template <typename Key, typename Value> struct CSVFilesParser
boost::spirit::istream_iterator sfirst(input_stream), slast; boost::spirit::istream_iterator sfirst(input_stream), slast;
Iterator first(sfirst), last(slast); Iterator first(sfirst), last(slast);
BOOST_ASSERT(file_id <= std::numeric_limits<decltype(Value::source)>::max());
ValueRule value_source = ValueRule value_source =
value_rule[qi::_val = qi::_1, boost::phoenix::bind(&Value::source, qi::_val) = file_id]; value_rule[qi::_val = qi::_1, boost::phoenix::bind(&Value::source, qi::_val) = file_id];
qi::rule<Iterator, std::pair<Key, Value>()> csv_line = qi::rule<Iterator, std::pair<Key, Value>()> csv_line =
@ -228,8 +228,9 @@ template <typename Key, typename Value> struct CSVFilesParser
return std::move(result); return std::move(result);
} }
KeyRule key_rule; const std::size_t start_index;
ValueRule value_rule; const KeyRule key_rule;
const ValueRule value_rule;
}; };
// Returns duration in deci-seconds // Returns duration in deci-seconds
@ -469,9 +470,10 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
util::Log() << "Reading " << graph_header.number_of_edges << " edges from the edge based graph"; util::Log() << "Reading " << graph_header.number_of_edges << " edges from the edge based graph";
auto segment_speed_lookup = CSVFilesParser<Segment, SpeedSource>( auto segment_speed_lookup = CSVFilesParser<Segment, SpeedSource>(
qi::ulong_long >> ',' >> qi::ulong_long, qi::uint_)(segment_speed_filenames); 1, qi::ulong_long >> ',' >> qi::ulong_long, qi::uint_)(segment_speed_filenames);
auto turn_penalty_lookup = CSVFilesParser<Turn, PenaltySource>( auto turn_penalty_lookup = CSVFilesParser<Turn, PenaltySource>(
1 + segment_speed_filenames.size(),
qi::ulong_long >> ',' >> qi::ulong_long >> ',' >> qi::ulong_long, qi::ulong_long >> ',' >> qi::ulong_long >> ',' >> qi::ulong_long,
qi::double_)(turn_penalty_filenames); qi::double_)(turn_penalty_filenames);
@ -567,70 +569,55 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
for (size_t i = 0; i < current_node.object_count; i++) for (size_t i = 0; i < current_node.object_count; i++)
{ {
const auto &leaf_object = current_node.objects[i]; const auto &leaf_object = current_node.objects[i];
extractor::QueryNode *u;
extractor::QueryNode *v;
const unsigned forward_begin = geometry_indices.at(leaf_object.packed_geometry_id); const auto forward_begin = geometry_indices.at(leaf_object.packed_geometry_id);
const auto current_fwd_weight = const auto u_index = forward_begin + leaf_object.fwd_segment_position;
geometry_fwd_weight_list[forward_begin + leaf_object.fwd_segment_position]; const auto v_index = forward_begin + leaf_object.fwd_segment_position + 1;
u = &(internal_to_external_node_map const extractor::QueryNode &u =
[geometry_node_list[forward_begin + leaf_object.fwd_segment_position]]); internal_to_external_node_map[geometry_node_list[u_index]];
v = &( const extractor::QueryNode &v =
internal_to_external_node_map internal_to_external_node_map[geometry_node_list[v_index]];
[geometry_node_list[forward_begin + leaf_object.fwd_segment_position + 1]]);
const double segment_length = util::coordinate_calculation::greatCircleDistance( const double segment_length = util::coordinate_calculation::greatCircleDistance(
util::Coordinate{u->lon, u->lat}, util::Coordinate{v->lon, v->lat}); util::Coordinate{u.lon, u.lat}, util::Coordinate{v.lon, v.lat});
if (auto value = segment_speed_lookup({u->node_id, v->node_id})) auto fwd_source = LUA_SOURCE, rev_source = LUA_SOURCE;
if (auto value = segment_speed_lookup({u.node_id, v.node_id}))
{ {
const auto current_fwd_weight = geometry_fwd_weight_list[u_index];
const auto new_segment_weight = GetNewWeight(*value, const auto new_segment_weight = GetNewWeight(*value,
segment_length, segment_length,
segment_speed_filenames, segment_speed_filenames,
current_fwd_weight, current_fwd_weight,
log_edge_updates_factor, log_edge_updates_factor,
u->node_id, u.node_id,
v->node_id); v.node_id);
geometry_fwd_weight_list[forward_begin + 1 + leaf_object.fwd_segment_position] = geometry_fwd_weight_list[v_index] = new_segment_weight;
new_segment_weight; geometry_datasource[v_index] = value->source;
geometry_datasource[forward_begin + 1 + leaf_object.fwd_segment_position] = fwd_source = value->source;
value->source;
// count statistics for logging
counters[value->source] += 1;
}
else
{
// count statistics for logging
counters[LUA_SOURCE] += 1;
} }
const auto current_rev_weight = if (auto value = segment_speed_lookup({v.node_id, u.node_id}))
geometry_rev_weight_list[forward_begin + leaf_object.fwd_segment_position];
if (auto value = segment_speed_lookup({v->node_id, u->node_id}))
{ {
const auto current_rev_weight = geometry_rev_weight_list[u_index];
const auto new_segment_weight = GetNewWeight(*value, const auto new_segment_weight = GetNewWeight(*value,
segment_length, segment_length,
segment_speed_filenames, segment_speed_filenames,
current_rev_weight, current_rev_weight,
log_edge_updates_factor, log_edge_updates_factor,
v->node_id, v.node_id,
u->node_id); u.node_id);
geometry_rev_weight_list[forward_begin + leaf_object.fwd_segment_position] =
new_segment_weight; geometry_rev_weight_list[u_index] = new_segment_weight;
geometry_datasource[forward_begin + leaf_object.fwd_segment_position] = geometry_datasource[u_index] = value->source;
value->source; rev_source = value->source;
}
// count statistics for logging // count statistics for logging
counters[value->source] += 1; counters[fwd_source] += 1;
} counters[rev_source] += 1;
else
{
counters[LUA_SOURCE] += 1;
}
} }
}); // parallel_for_each }); // parallel_for_each
@ -713,89 +700,69 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
throw util::exception(message + SOURCE_REF); throw util::exception(message + SOURCE_REF);
} }
datasource_stream << "lua profile" << std::endl; datasource_stream << "lua profile" << std::endl;
for (auto const &name : segment_speed_filenames)
{
// Only write the filename, without path or extension. // Only write the filename, without path or extension.
// This prevents information leakage, and keeps names short // This prevents information leakage, and keeps names short
// for rendering in the debug tiles. // for rendering in the debug tiles.
const boost::filesystem::path p(name); for (auto const &name : segment_speed_filenames)
datasource_stream << p.stem().string() << std::endl; {
datasource_stream << boost::filesystem::path(name).stem().string() << std::endl;
} }
}; };
tbb::parallel_invoke(maybe_save_geometries, save_datasource_indexes, save_datastore_names); tbb::parallel_invoke(maybe_save_geometries, save_datasource_indexes, save_datastore_names);
auto penaltyblock = reinterpret_cast<const extractor::lookup::PenaltyBlock *>( auto turn_penalty_blocks_ptr = reinterpret_cast<const extractor::lookup::PenaltyBlock *>(
edge_penalty_region.get_address()); edge_penalty_region.get_address());
BOOST_ASSERT(is_aligned<extractor::lookup::PenaltyBlock>(penaltyblock)); BOOST_ASSERT(is_aligned<extractor::lookup::PenaltyBlock>(turn_penalty_blocks_ptr));
auto edge_based_edge_ptr = reinterpret_cast<extractor::EdgeBasedEdge *>( auto edge_based_edge_ptr = reinterpret_cast<extractor::EdgeBasedEdge *>(
reinterpret_cast<char *>(edge_based_graph_region.get_address()) + reinterpret_cast<char *>(edge_based_graph_region.get_address()) +
sizeof(EdgeBasedGraphHeader)); sizeof(EdgeBasedGraphHeader));
BOOST_ASSERT(is_aligned<extractor::EdgeBasedEdge>(edge_based_edge_ptr)); BOOST_ASSERT(is_aligned<extractor::EdgeBasedEdge>(edge_based_edge_ptr));
const auto edge_based_edge_last = reinterpret_cast<extractor::EdgeBasedEdge *>(
reinterpret_cast<char *>(edge_based_graph_region.get_address()) +
sizeof(EdgeBasedGraphHeader) +
sizeof(extractor::EdgeBasedEdge) * graph_header.number_of_edges);
auto edge_segment_byte_ptr = reinterpret_cast<const char *>(edge_segment_region.get_address()); auto edge_segment_byte_ptr = reinterpret_cast<const char *>(edge_segment_region.get_address());
while (edge_based_edge_ptr != edge_based_edge_last) for (std::uint64_t edge_index = 0; edge_index < graph_header.number_of_edges; ++edge_index)
{ {
// Make a copy of the data from the memory map // Make a copy of the data from the memory map
extractor::EdgeBasedEdge inbuffer = *edge_based_edge_ptr; extractor::EdgeBasedEdge inbuffer = edge_based_edge_ptr[edge_index];
edge_based_edge_ptr++;
if (update_edge_weights || update_turn_penalties) if (update_edge_weights || update_turn_penalties)
{ {
bool skip_this_edge = false; using extractor::lookup::SegmentHeaderBlock;
using extractor::lookup::SegmentBlock;
auto header = reinterpret_cast<const extractor::lookup::SegmentHeaderBlock *>( auto header = reinterpret_cast<const SegmentHeaderBlock *>(edge_segment_byte_ptr);
edge_segment_byte_ptr); BOOST_ASSERT(is_aligned<SegmentHeaderBlock>(header));
BOOST_ASSERT(is_aligned<extractor::lookup::SegmentHeaderBlock>(header)); edge_segment_byte_ptr += sizeof(SegmentHeaderBlock);
edge_segment_byte_ptr += sizeof(extractor::lookup::SegmentHeaderBlock);
auto previous_osm_node_id = header->previous_osm_node_id; auto first = reinterpret_cast<const SegmentBlock *>(edge_segment_byte_ptr);
BOOST_ASSERT(is_aligned<SegmentBlock>(first));
edge_segment_byte_ptr += sizeof(SegmentBlock) * (header->num_osm_nodes - 1);
auto last = reinterpret_cast<const SegmentBlock *>(edge_segment_byte_ptr);
// Find a segment with zero speed and simultaneously compute the new edge weight
EdgeWeight new_weight = 0; EdgeWeight new_weight = 0;
int compressed_edge_nodes = static_cast<int>(header->num_osm_nodes); auto osm_node_id = header->previous_osm_node_id;
bool skip_edge =
auto segmentblocks = std::find_if(first, last, [&](const auto &segment) {
reinterpret_cast<const extractor::lookup::SegmentBlock *>(edge_segment_byte_ptr); auto segment_weight = segment.segment_weight;
BOOST_ASSERT(is_aligned<extractor::lookup::SegmentBlock>(segmentblocks)); if (auto value = segment_speed_lookup({osm_node_id, segment.this_osm_node_id}))
edge_segment_byte_ptr +=
sizeof(extractor::lookup::SegmentBlock) * (header->num_osm_nodes - 1);
const auto num_segments = header->num_osm_nodes - 1;
for (auto i : util::irange<std::size_t>(0, num_segments))
{
if (auto value = segment_speed_lookup(
{previous_osm_node_id, segmentblocks[i].this_osm_node_id}))
{
if (value->speed > 0)
{
const auto new_segment_weight =
ConvertToDuration(segmentblocks[i].segment_length, value->speed);
new_weight += new_segment_weight;
}
else
{ {
// If we hit a 0-speed edge, then it's effectively not traversible. // If we hit a 0-speed edge, then it's effectively not traversible.
// We don't want to include it in the edge_based_edge_list, so // We don't want to include it in the edge_based_edge_list.
// we set a flag and `continue` the parent loop as soon as we can. if (value->speed == 0)
// This would be a perfect place to use `goto`, but Patrick vetoed it. return true;
skip_this_edge = true;
break; segment_weight = ConvertToDuration(segment.segment_length, value->speed);
}
}
else
{
// If no lookup found, use the original weight value for this segment
new_weight += segmentblocks[i].segment_weight;
} }
previous_osm_node_id = segmentblocks[i].this_osm_node_id; // Update the edge weight and the next OSM node ID
} osm_node_id = segment.this_osm_node_id;
new_weight += segment_weight;
return false;
}) != last;
// Update the node-weight cache. This is the weight of the edge-based-node only, // Update the node-weight cache. This is the weight of the edge-based-node only,
// it doesn't include the turn. We may visit the same node multiple times, but // it doesn't include the turn. We may visit the same node multiple times, but
@ -804,35 +771,29 @@ EdgeID Contractor::LoadEdgeExpandedGraph(
// We found a zero-speed edge, so we'll skip this whole edge-based-edge which // We found a zero-speed edge, so we'll skip this whole edge-based-edge which
// effectively removes it from the routing network. // effectively removes it from the routing network.
if (skip_this_edge) if (skip_edge)
{
penaltyblock++;
continue; continue;
}
if (auto value = turn_penalty_lookup( // Get the turn penalty and update to the new value if required
{penaltyblock->from_id, penaltyblock->via_id, penaltyblock->to_id})) const auto &turn_block = turn_penalty_blocks_ptr[edge_index];
EdgeWeight new_turn_penalty = turn_block.fixed_penalty;
if (auto value = turn_penalty_lookup(turn_block))
{ {
int new_turn_weight = static_cast<int>(value->penalty * 10); new_turn_penalty = static_cast<EdgeWeight>(value->penalty * 10);
if (new_turn_weight + new_weight < compressed_edge_nodes) if (new_weight + new_turn_penalty < static_cast<EdgeWeight>(header->num_osm_nodes))
{ {
util::Log(logWARNING) << "turn penalty " << value->penalty << " for turn " util::Log(logWARNING)
<< penaltyblock->from_id << ", " << penaltyblock->via_id << "turn penalty " << value->penalty << " for turn " << turn_block.from_id
<< ", " << penaltyblock->to_id << "," << turn_block.via_id << "," << turn_block.to_id
<< " is too negative: clamping turn weight to " << " is too negative: clamping turn weight to " << header->num_osm_nodes;
<< compressed_edge_nodes;
new_turn_penalty = header->num_osm_nodes - new_weight;
}
} }
inbuffer.weight = std::max(new_turn_weight + new_weight, compressed_edge_nodes); // Update edge weight
} inbuffer.weight = new_weight + new_turn_penalty;
else
{
inbuffer.weight = penaltyblock->fixed_penalty + new_weight;
}
// Increment the pointer
penaltyblock++;
} }
edge_based_edge_list.emplace_back(std::move(inbuffer)); edge_based_edge_list.emplace_back(std::move(inbuffer));