Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Expose concurrency in saving files

Browse Source
This commit is contained in:
Daniel J. Hofmann 2016-05-17 13:50:29 -04:00 committed by Patrick Niklaus
parent 0a73737f69
commit 8291be375a
No known key found for this signature in database
GPG Key ID: E426891B5F978B1B
1 changed files with 118 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
src/contractor/contractor.cpp
View File

@ -361,8 +361,6 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
// Now, we iterate over all the segments stored in the StaticRTree, updating // Now, we iterate over all the segments stored in the StaticRTree, updating
// the packed geometry weights in the `.geometries` file (note: we do not // the packed geometry weights in the `.geometries` file (note: we do not
// update the RTree itself, we just use the leaf nodes to iterate over all segments) // update the RTree itself, we just use the leaf nodes to iterate over all segments)
{
using LeafNode = util::StaticRTree<extractor::EdgeBasedNode>::LeafNode; using LeafNode = util::StaticRTree<extractor::EdgeBasedNode>::LeafNode;
std::ifstream leaf_node_file(rtree_leaf_filename, std::ifstream leaf_node_file(rtree_leaf_filename,
@ -393,8 +391,8 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
if (leaf_object.fwd_segment_position == 0) if (leaf_object.fwd_segment_position == 0)
{ {
u = &(internal_to_external_node_map[leaf_object.u]); u = &(internal_to_external_node_map[leaf_object.u]);
v = &(internal_to_external_node_map[m_geometry_list[forward_begin] v = &(
.node_id]); internal_to_external_node_map[m_geometry_list[forward_begin].node_id]);
} }
else else
{ {
@ -403,27 +401,24 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
leaf_object.fwd_segment_position - 1] leaf_object.fwd_segment_position - 1]
.node_id]); .node_id]);
v = &(internal_to_external_node_map v = &(internal_to_external_node_map
[m_geometry_list[forward_begin + [m_geometry_list[forward_begin + leaf_object.fwd_segment_position]
leaf_object.fwd_segment_position]
.node_id]); .node_id]);
} }
const double segment_length = const double segment_length = util::coordinate_calculation::greatCircleDistance(
util::coordinate_calculation::greatCircleDistance(
util::Coordinate{u->lon, u->lat}, util::Coordinate{v->lon, v->lat}); util::Coordinate{u->lon, u->lat}, util::Coordinate{v->lon, v->lat});
auto forward_speed_iter = auto forward_speed_iter =
segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id)); segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id));
if (forward_speed_iter != segment_speed_lookup.end()) if (forward_speed_iter != segment_speed_lookup.end())
{ {
int new_segment_weight = int new_segment_weight = std::max(
std::max(1, static_cast<int>(std::floor( 1,
(segment_length * 10.) / static_cast<int>(std::floor(
(forward_speed_iter->second.first / 3.6) + (segment_length * 10.) / (forward_speed_iter->second.first / 3.6) +
.5))); .5)));
m_geometry_list[forward_begin + leaf_object.fwd_segment_position] m_geometry_list[forward_begin + leaf_object.fwd_segment_position].weight =
.weight = new_segment_weight; new_segment_weight;
m_geometry_datasource[forward_begin + m_geometry_datasource[forward_begin + leaf_object.fwd_segment_position] =
leaf_object.fwd_segment_position] =
forward_speed_iter->second.second; forward_speed_iter->second.second;
} }
} }
@ -439,31 +434,29 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
if (rev_segment_position == 0) if (rev_segment_position == 0)
{ {
u = &(internal_to_external_node_map[leaf_object.v]); u = &(internal_to_external_node_map[leaf_object.v]);
v = &(internal_to_external_node_map[m_geometry_list[reverse_begin] v = &(
.node_id]); internal_to_external_node_map[m_geometry_list[reverse_begin].node_id]);
} }
else else
{ {
u = &(internal_to_external_node_map u = &(
[m_geometry_list[reverse_begin + rev_segment_position - 1]
.node_id]);
v = &(
internal_to_external_node_map[m_geometry_list[reverse_begin + internal_to_external_node_map[m_geometry_list[reverse_begin +
rev_segment_position] rev_segment_position - 1]
.node_id]); .node_id]);
v = &(internal_to_external_node_map
[m_geometry_list[reverse_begin + rev_segment_position].node_id]);
} }
const double segment_length = const double segment_length = util::coordinate_calculation::greatCircleDistance(
util::coordinate_calculation::greatCircleDistance(
util::Coordinate{u->lon, u->lat}, util::Coordinate{v->lon, v->lat}); util::Coordinate{u->lon, u->lat}, util::Coordinate{v->lon, v->lat});
auto reverse_speed_iter = auto reverse_speed_iter =
segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id)); segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id));
if (reverse_speed_iter != segment_speed_lookup.end()) if (reverse_speed_iter != segment_speed_lookup.end())
{ {
int new_segment_weight = int new_segment_weight = std::max(
std::max(1, static_cast<int>(std::floor( 1,
(segment_length * 10.) / static_cast<int>(std::floor(
(reverse_speed_iter->second.first / 3.6) + (segment_length * 10.) / (reverse_speed_iter->second.first / 3.6) +
.5))); .5)));
m_geometry_list[reverse_begin + rev_segment_position].weight = m_geometry_list[reverse_begin + rev_segment_position].weight =
new_segment_weight; new_segment_weight;
@ -476,8 +469,11 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
} }
} }
const auto maybe_save_geometries = [&] {
if (!(update_edge_weights || update_turn_penalties))
return;
// Now save out the updated compressed geometries // Now save out the updated compressed geometries
{
std::ofstream geometry_stream(geometry_filename, std::ios::binary); std::ofstream geometry_stream(geometry_filename, std::ios::binary);
if (!geometry_stream) if (!geometry_stream)
{ {
@ -485,8 +481,7 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
} }
const unsigned number_of_indices = m_geometry_indices.size(); const unsigned number_of_indices = m_geometry_indices.size();
const unsigned number_of_compressed_geometries = m_geometry_list.size(); const unsigned number_of_compressed_geometries = m_geometry_list.size();
geometry_stream.write(reinterpret_cast<const char *>(&number_of_indices), geometry_stream.write(reinterpret_cast<const char *>(&number_of_indices), sizeof(unsigned));
sizeof(unsigned));
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_indices[0])), geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_indices[0])),
number_of_indices * sizeof(unsigned)); number_of_indices * sizeof(unsigned));
geometry_stream.write(reinterpret_cast<const char *>(&number_of_compressed_geometries), geometry_stream.write(reinterpret_cast<const char *>(&number_of_compressed_geometries),
@ -494,10 +489,9 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_list[0])), geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_list[0])),
number_of_compressed_geometries * number_of_compressed_geometries *
sizeof(extractor::CompressedEdgeContainer::CompressedEdge)); sizeof(extractor::CompressedEdgeContainer::CompressedEdge));
} };
}
{ const auto save_datasource_indexes = [&] {
std::ofstream datasource_stream(datasource_indexes_filename, std::ios::binary); std::ofstream datasource_stream(datasource_indexes_filename, std::ios::binary);
if (!datasource_stream) if (!datasource_stream)
{ {
@ -511,9 +505,9 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
datasource_stream.write(reinterpret_cast<char *>(&(m_geometry_datasource[0])), datasource_stream.write(reinterpret_cast<char *>(&(m_geometry_datasource[0])),
number_of_datasource_entries * sizeof(uint8_t)); number_of_datasource_entries * sizeof(uint8_t));
} }
} };
{ const auto save_datastore_names = [&] {
std::ofstream datasource_stream(datasource_names_filename, std::ios::binary); std::ofstream datasource_stream(datasource_names_filename, std::ios::binary);
if (!datasource_stream) if (!datasource_stream)
{ {
@ -524,7 +518,9 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
{ {
datasource_stream << name << std::endl; datasource_stream << name << std::endl;
} }
} };
tbb::parallel_invoke(maybe_save_geometries, save_datasource_indexes, save_datastore_names);
// TODO: can we read this in bulk? util::DeallocatingVector isn't necessarily // TODO: can we read this in bulk? util::DeallocatingVector isn't necessarily
// all stored contiguously // all stored contiguously