Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Make forward/reverse weight/offset calculated at query time,

Browse Source 
rather than being cached in the StaticRTree.  This means we
can freely apply traffic data and not have stale values lying
around.  It reduces the size of the RTree on disk, at the expense
of some additional data in RAM.
This commit is contained in:
Daniel Patterson
2016-01-29 17:52:20 -08:00
parent 03d360b7bf
commit 49441fe204
24 changed files with 760 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/extractor/compressed_edge_container.cpp
+66 -11
View File
@@ -59,7 +59,7 @@ void CompressedEdgeContainer::SerializeInternalVector(const std::string &path) c
{
geometry_out_stream.write((char *)&prefix_sum_of_list_indices, sizeof(unsigned));
const std::vector<CompressedNode> &current_vector = elem;
const std::vector<CompressedEdge> &current_vector = elem;
const unsigned unpacked_size = current_vector.size();
BOOST_ASSERT(std::numeric_limits<unsigned>::max() != unpacked_size);
prefix_sum_of_list_indices += unpacked_size;
@@ -74,13 +74,13 @@ void CompressedEdgeContainer::SerializeInternalVector(const std::string &path) c
// write compressed geometries
for (auto &elem : m_compressed_geometries)
{
const std::vector<CompressedNode> &current_vector = elem;
const std::vector<CompressedEdge> &current_vector = elem;
const unsigned unpacked_size = current_vector.size();
control_sum += unpacked_size;
BOOST_ASSERT(std::numeric_limits<unsigned>::max() != unpacked_size);
for (const CompressedNode current_node : current_vector)
for (const auto & current_node : current_vector)
{
geometry_out_stream.write((char *)&(current_node.first), sizeof(NodeID));
geometry_out_stream.write((char *)&(current_node), sizeof(CompressedEdge));
}
}
BOOST_ASSERT(control_sum == prefix_sum_of_list_indices);
@@ -88,6 +88,14 @@ void CompressedEdgeContainer::SerializeInternalVector(const std::string &path) c
geometry_out_stream.close();
}
// Adds info for a compressed edge to the container. edge_id_2
// has been removed from the graph, so we have to save These edges/nodes
// have already been trimmed from the graph, this function just stores
// the original data for unpacking later.
//
// edge_id_1 edge_id_2
// ----------> via_node_id -----------> target_node_id
// weight_1 weight_2
void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1,
const EdgeID edge_id_2,
const NodeID via_node_id,
@@ -131,13 +139,13 @@ void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1,
BOOST_ASSERT(edge_bucket_id1 == GetPositionForID(edge_id_1));
BOOST_ASSERT(edge_bucket_id1 < m_compressed_geometries.size());
std::vector<CompressedNode> &edge_bucket_list1 = m_compressed_geometries[edge_bucket_id1];
std::vector<CompressedEdge> &edge_bucket_list1 = m_compressed_geometries[edge_bucket_id1];
// note we don't save the start coordinate: it is implicitly given by edge 1
// weight1 is the distance to the (currently) last coordinate in the bucket
if (edge_bucket_list1.empty())
{
edge_bucket_list1.emplace_back(via_node_id, weight1);
edge_bucket_list1.emplace_back(CompressedEdge { via_node_id, weight1 });
}
BOOST_ASSERT(0 < edge_bucket_list1.size());
@@ -149,7 +157,7 @@ void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1,
const unsigned list_to_remove_index = GetPositionForID(edge_id_2);
BOOST_ASSERT(list_to_remove_index < m_compressed_geometries.size());
std::vector<CompressedNode> &edge_bucket_list2 =
std::vector<CompressedEdge> &edge_bucket_list2 =
m_compressed_geometries[list_to_remove_index];
// found an existing list, append it to the list of edge_id_1
@@ -168,10 +176,57 @@ void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1,
else
{
// we are certain that the second edge is atomic.
edge_bucket_list1.emplace_back(target_node_id, weight2);
edge_bucket_list1.emplace_back( CompressedEdge { target_node_id, weight2 });
}
}
void CompressedEdgeContainer::AddUncompressedEdge(const EdgeID edge_id,
const NodeID target_node_id,
const EdgeWeight weight)
{
// remove super-trivial geometries
BOOST_ASSERT(SPECIAL_EDGEID != edge_id);
BOOST_ASSERT(SPECIAL_NODEID != target_node_id);
BOOST_ASSERT(INVALID_EDGE_WEIGHT != weight);
// There should be no entry for uncompressed edges
BOOST_ASSERT(!HasEntryForID(edge_id));
// Add via node id. List is created if it does not exist
if (!HasEntryForID(edge_id))
{
// create a new entry in the map
if (0 == m_free_list.size())
{
// make sure there is a place to put the entries
IncreaseFreeList();
}
BOOST_ASSERT(!m_free_list.empty());
m_edge_id_to_list_index_map[edge_id] = m_free_list.back();
m_free_list.pop_back();
}
// find bucket index
const auto iter = m_edge_id_to_list_index_map.find(edge_id);
BOOST_ASSERT(iter != m_edge_id_to_list_index_map.end());
const unsigned edge_bucket_id = iter->second;
BOOST_ASSERT(edge_bucket_id == GetPositionForID(edge_id));
BOOST_ASSERT(edge_bucket_id < m_compressed_geometries.size());
std::vector<CompressedEdge> &edge_bucket_list = m_compressed_geometries[edge_bucket_id];
// We're adding uncompressed edges, there should be no entry
BOOST_ASSERT(edge_bucket_list.empty());
// note we don't save the start coordinate: it is implicitly given by edge_id
// weight is the distance to the (currently) last coordinate in the bucket
if (edge_bucket_list.empty())
{
edge_bucket_list.emplace_back(CompressedEdge { target_node_id, weight });
}
}
void CompressedEdgeContainer::PrintStatistics() const
{
const uint64_t compressed_edges = m_compressed_geometries.size();
@@ -180,7 +235,7 @@ void CompressedEdgeContainer::PrintStatistics() const
uint64_t compressed_geometries = 0;
uint64_t longest_chain_length = 0;
for (const std::vector<CompressedNode> &current_vector : m_compressed_geometries)
for (const std::vector<CompressedEdge> &current_vector : m_compressed_geometries)
{
compressed_geometries += current_vector.size();
longest_chain_length = std::max(longest_chain_length, (uint64_t)current_vector.size());
@@ -207,13 +262,13 @@ NodeID CompressedEdgeContainer::GetFirstEdgeTargetID(const EdgeID edge_id) const
{
const auto &bucket = GetBucketReference(edge_id);
BOOST_ASSERT(bucket.size() >= 2);
return bucket.front().first;
return bucket.front().node_id;
}
NodeID CompressedEdgeContainer::GetLastEdgeSourceID(const EdgeID edge_id) const
{
const auto &bucket = GetBucketReference(edge_id);
BOOST_ASSERT(bucket.size() >= 2);
return bucket[bucket.size() - 2].first;
return bucket[bucket.size() - 2].node_id;
}
}
}
src/extractor/edge_based_graph_factory.cpp
+30 -30
View File
@@ -45,7 +45,7 @@ const double constexpr DESIRED_SEGMENT_LENGTH = 10.;
EdgeBasedGraphFactory::EdgeBasedGraphFactory(
std::shared_ptr<util::NodeBasedDynamicGraph> node_based_graph,
const CompressedEdgeContainer &compressed_edge_container,
CompressedEdgeContainer &compressed_edge_container,
const std::unordered_set<NodeID> &barrier_nodes,
const std::unordered_set<NodeID> &traffic_lights,
std::shared_ptr<const RestrictionMap> restriction_map,
@@ -138,8 +138,8 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI
// reconstruct bidirectional edge with individual weights and put each into the NN index
std::vector<int> forward_dist_prefix_sum(forward_geometry.size(), 0);
std::vector<int> reverse_dist_prefix_sum(reverse_geometry.size(), 0);
std::vector<int> forward_offsets(forward_geometry.size(), 0);
std::vector<int> reverse_offsets(reverse_geometry.size(), 0);
// quick'n'dirty prefix sum as std::partial_sum needs addtional casts
// TODO: move to lambda function with C++11
@@ -147,8 +147,8 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI
for (const auto i : util::irange(0u, geometry_size))
{
forward_dist_prefix_sum[i] = temp_sum;
temp_sum += forward_geometry[i].second;
forward_offsets[i] = temp_sum;
temp_sum += forward_geometry[i].weight;
BOOST_ASSERT(forward_data.distance >= temp_sum);
}
@@ -156,8 +156,8 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI
temp_sum = 0;
for (const auto i : util::irange(0u, geometry_size))
{
temp_sum += reverse_geometry[reverse_geometry.size() - 1 - i].second;
reverse_dist_prefix_sum[i] = reverse_data.distance - temp_sum;
temp_sum += reverse_geometry[reverse_geometry.size() - 1 - i].weight;
reverse_offsets[i] = reverse_data.distance - temp_sum;
// BOOST_ASSERT(reverse_data.distance >= temp_sum);
}
@@ -167,24 +167,22 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI
for (const auto i : util::irange(0u, geometry_size))
{
BOOST_ASSERT(current_edge_source_coordinate_id ==
reverse_geometry[geometry_size - 1 - i].first);
const NodeID current_edge_target_coordinate_id = forward_geometry[i].first;
reverse_geometry[geometry_size - 1 - i].node_id);
const NodeID current_edge_target_coordinate_id = forward_geometry[i].node_id;
BOOST_ASSERT(current_edge_target_coordinate_id != current_edge_source_coordinate_id);
// build edges
m_edge_based_node_list.emplace_back(
forward_data.edge_id, reverse_data.edge_id, current_edge_source_coordinate_id,
current_edge_target_coordinate_id, forward_data.name_id, forward_geometry[i].second,
reverse_geometry[geometry_size - 1 - i].second, forward_dist_prefix_sum[i],
reverse_dist_prefix_sum[i], m_compressed_edge_container.GetPositionForID(edge_id_1),
current_edge_target_coordinate_id, forward_data.name_id,
m_compressed_edge_container.GetPositionForID(edge_id_1),
m_compressed_edge_container.GetPositionForID(edge_id_2),
false, INVALID_COMPONENTID, i, forward_data.travel_mode, reverse_data.travel_mode);
m_edge_based_node_is_startpoint.push_back(forward_data.startpoint ||
reverse_data.startpoint);
current_edge_source_coordinate_id = current_edge_target_coordinate_id;
BOOST_ASSERT(m_edge_based_node_list.back().IsCompressed());
BOOST_ASSERT(node_u != m_edge_based_node_list.back().u ||
node_v != m_edge_based_node_list.back().v);
@@ -193,7 +191,6 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI
}
BOOST_ASSERT(current_edge_source_coordinate_id == node_v);
BOOST_ASSERT(m_edge_based_node_list.back().IsCompressed());
}
else
{
@@ -220,13 +217,16 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI
BOOST_ASSERT(forward_data.edge_id != SPECIAL_NODEID ||
reverse_data.edge_id != SPECIAL_NODEID);
m_compressed_edge_container.AddUncompressedEdge(edge_id_1, node_v, forward_data.distance);
m_compressed_edge_container.AddUncompressedEdge(edge_id_2, node_u, reverse_data.distance);
m_edge_based_node_list.emplace_back(
forward_data.edge_id, reverse_data.edge_id, node_u, node_v, forward_data.name_id,
forward_data.distance, reverse_data.distance, 0, 0, SPECIAL_EDGEID, false,
INVALID_COMPONENTID, 0, forward_data.travel_mode, reverse_data.travel_mode);
m_compressed_edge_container.GetPositionForID(edge_id_1),
m_compressed_edge_container.GetPositionForID(edge_id_2),
false, INVALID_COMPONENTID, 0, forward_data.travel_mode, reverse_data.travel_mode);
m_edge_based_node_is_startpoint.push_back(forward_data.startpoint ||
reverse_data.startpoint);
BOOST_ASSERT(!m_edge_based_node_list.back().IsCompressed());
}
}
@@ -500,7 +500,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
for (auto target_node : node_based_edges)
{
const QueryNode &from = m_node_info_list[previous];
const QueryNode &to = m_node_info_list[target_node.first];
const QueryNode &to = m_node_info_list[target_node.node_id];
const double segment_length =
util::coordinate_calculation::greatCircleDistance(
from.lat, from.lon, to.lat, to.lon);
@@ -510,9 +510,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
edge_segment_file.write(reinterpret_cast<const char *>(&segment_length),
sizeof(segment_length));
edge_segment_file.write(
reinterpret_cast<const char *>(&target_node.second),
sizeof(target_node.second));
previous = target_node.first;
reinterpret_cast<const char *>(&target_node.weight),
sizeof(target_node.weight));
previous = target_node.node_id;
}
}
else
@@ -1131,17 +1131,17 @@ QueryNode EdgeBasedGraphFactory::getRepresentativeCoordinate(const NodeID src,
for (auto itr = geometry.rbegin(), end = geometry.rend(); itr != end; ++itr)
{
const auto compressed_node = *itr;
cur = util::FixedPointCoordinate(m_node_info_list[compressed_node.first].lat,
m_node_info_list[compressed_node.first].lon);
cur = util::FixedPointCoordinate(m_node_info_list[compressed_node.node_id].lat,
m_node_info_list[compressed_node.node_id].lon);
this_dist = util::coordinate_calculation::haversineDistance(prev, cur);
if (dist + this_dist > DESIRED_SEGMENT_LENGTH)
{
return selectBestCandidate(compressed_node.first, dist + this_dist, prev_id,
return selectBestCandidate(compressed_node.node_id, dist + this_dist, prev_id,
dist);
}
dist += this_dist;
prev = cur;
prev_id = compressed_node.first;
prev_id = compressed_node.node_id;
}
cur = util::FixedPointCoordinate(m_node_info_list[src].lat, m_node_info_list[src].lon);
this_dist = util::coordinate_calculation::haversineDistance(prev, cur);
@@ -1152,17 +1152,17 @@ QueryNode EdgeBasedGraphFactory::getRepresentativeCoordinate(const NodeID src,
for (auto itr = geometry.begin(), end = geometry.end(); itr != end; ++itr)
{
const auto compressed_node = *itr;
cur = util::FixedPointCoordinate(m_node_info_list[compressed_node.first].lat,
m_node_info_list[compressed_node.first].lon);
cur = util::FixedPointCoordinate(m_node_info_list[compressed_node.node_id].lat,
m_node_info_list[compressed_node.node_id].lon);
this_dist = util::coordinate_calculation::haversineDistance(prev, cur);
if (dist + this_dist > DESIRED_SEGMENT_LENGTH)
{
return selectBestCandidate(compressed_node.first, dist + this_dist, prev_id,
return selectBestCandidate(compressed_node.node_id, dist + this_dist, prev_id,
dist);
}
dist += this_dist;
prev = cur;
prev_id = compressed_node.first;
prev_id = compressed_node.node_id;
}
cur = util::FixedPointCoordinate(m_node_info_list[tgt].lat, m_node_info_list[tgt].lon);
this_dist = util::coordinate_calculation::haversineDistance(prev, cur);
src/extractor/extractor.cpp
+6 -1
View File
@@ -522,7 +522,6 @@ Extractor::BuildEdgeExpandedGraph(std::vector<QueryNode> &internal_to_external_n
std::const_pointer_cast<RestrictionMap const>(restriction_map),
internal_to_external_node_map, speed_profile);
compressed_edge_container.SerializeInternalVector(config.geometry_output_path);
edge_based_graph_factory.Run(config.edge_output_path, lua_state,
config.edge_segment_lookup_path, config.edge_penalty_path,
@@ -532,6 +531,12 @@ Extractor::BuildEdgeExpandedGraph(std::vector<QueryNode> &internal_to_external_n
config.debug_turns_path
#endif
);
// Note: this needs to be done *after* the edge-based-graph-factory runs,
// becase it will insert all the uncompressable segments into the compressed
// edge container (necessary for later use)
compressed_edge_container.SerializeInternalVector(config.geometry_output_path);
lua_close(lua_state);
edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list);