Expose concurrency in saving files
This commit is contained in:
Daniel J. Hofmann
Patrick Niklaus
parent
0a73737f69
commit
8291be375a
@ -361,143 +361,137 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
|
|||||||
// Now, we iterate over all the segments stored in the StaticRTree, updating
|
// Now, we iterate over all the segments stored in the StaticRTree, updating
|
||||||
// the packed geometry weights in the `.geometries` file (note: we do not
|
// the packed geometry weights in the `.geometries` file (note: we do not
|
||||||
// update the RTree itself, we just use the leaf nodes to iterate over all segments)
|
// update the RTree itself, we just use the leaf nodes to iterate over all segments)
|
||||||
|
using LeafNode = util::StaticRTree<extractor::EdgeBasedNode>::LeafNode;
|
||||||
|
|
||||||
|
std::ifstream leaf_node_file(rtree_leaf_filename,
|
||||||
|
std::ios::binary | std::ios::in | std::ios::ate);
|
||||||
|
if (!leaf_node_file)
|
||||||
{
|
{
|
||||||
|
throw util::exception("Failed to open " + rtree_leaf_filename);
|
||||||
|
}
|
||||||
|
std::size_t leaf_nodes_count = leaf_node_file.tellg() / sizeof(LeafNode);
|
||||||
|
leaf_node_file.seekg(0, std::ios::beg);
|
||||||
|
|
||||||
using LeafNode = util::StaticRTree<extractor::EdgeBasedNode>::LeafNode;
|
LeafNode current_node;
|
||||||
|
while (leaf_nodes_count > 0)
|
||||||
|
{
|
||||||
|
leaf_node_file.read(reinterpret_cast<char *>(¤t_node), sizeof(current_node));
|
||||||
|
|
||||||
std::ifstream leaf_node_file(rtree_leaf_filename,
|
for (size_t i = 0; i < current_node.object_count; i++)
|
||||||
std::ios::binary | std::ios::in | std::ios::ate);
|
|
||||||
if (!leaf_node_file)
|
|
||||||
{
|
{
|
||||||
throw util::exception("Failed to open " + rtree_leaf_filename);
|
auto &leaf_object = current_node.objects[i];
|
||||||
}
|
extractor::QueryNode *u;
|
||||||
std::size_t leaf_nodes_count = leaf_node_file.tellg() / sizeof(LeafNode);
|
extractor::QueryNode *v;
|
||||||
leaf_node_file.seekg(0, std::ios::beg);
|
|
||||||
|
|
||||||
LeafNode current_node;
|
if (leaf_object.forward_packed_geometry_id != SPECIAL_EDGEID)
|
||||||
while (leaf_nodes_count > 0)
|
|
||||||
{
|
|
||||||
leaf_node_file.read(reinterpret_cast<char *>(¤t_node), sizeof(current_node));
|
|
||||||
|
|
||||||
for (size_t i = 0; i < current_node.object_count; i++)
|
|
||||||
{
|
{
|
||||||
auto &leaf_object = current_node.objects[i];
|
const unsigned forward_begin =
|
||||||
extractor::QueryNode *u;
|
m_geometry_indices.at(leaf_object.forward_packed_geometry_id);
|
||||||
extractor::QueryNode *v;
|
|
||||||
|
|
||||||
if (leaf_object.forward_packed_geometry_id != SPECIAL_EDGEID)
|
if (leaf_object.fwd_segment_position == 0)
|
||||||
{
|
{
|
||||||
const unsigned forward_begin =
|
u = &(internal_to_external_node_map[leaf_object.u]);
|
||||||
m_geometry_indices.at(leaf_object.forward_packed_geometry_id);
|
v = &(
|
||||||
|
internal_to_external_node_map[m_geometry_list[forward_begin].node_id]);
|
||||||
if (leaf_object.fwd_segment_position == 0)
|
|
||||||
{
|
|
||||||
u = &(internal_to_external_node_map[leaf_object.u]);
|
|
||||||
v = &(internal_to_external_node_map[m_geometry_list[forward_begin]
|
|
||||||
.node_id]);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
u = &(internal_to_external_node_map
|
|
||||||
[m_geometry_list[forward_begin +
|
|
||||||
leaf_object.fwd_segment_position - 1]
|
|
||||||
.node_id]);
|
|
||||||
v = &(internal_to_external_node_map
|
|
||||||
[m_geometry_list[forward_begin +
|
|
||||||
leaf_object.fwd_segment_position]
|
|
||||||
.node_id]);
|
|
||||||
}
|
|
||||||
const double segment_length =
|
|
||||||
util::coordinate_calculation::greatCircleDistance(
|
|
||||||
util::Coordinate{u->lon, u->lat}, util::Coordinate{v->lon, v->lat});
|
|
||||||
|
|
||||||
auto forward_speed_iter =
|
|
||||||
segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id));
|
|
||||||
if (forward_speed_iter != segment_speed_lookup.end())
|
|
||||||
{
|
|
||||||
int new_segment_weight =
|
|
||||||
std::max(1, static_cast<int>(std::floor(
|
|
||||||
(segment_length * 10.) /
|
|
||||||
(forward_speed_iter->second.first / 3.6) +
|
|
||||||
.5)));
|
|
||||||
m_geometry_list[forward_begin + leaf_object.fwd_segment_position]
|
|
||||||
.weight = new_segment_weight;
|
|
||||||
m_geometry_datasource[forward_begin +
|
|
||||||
leaf_object.fwd_segment_position] =
|
|
||||||
forward_speed_iter->second.second;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (leaf_object.reverse_packed_geometry_id != SPECIAL_EDGEID)
|
else
|
||||||
{
|
{
|
||||||
const unsigned reverse_begin =
|
u = &(internal_to_external_node_map
|
||||||
m_geometry_indices.at(leaf_object.reverse_packed_geometry_id);
|
[m_geometry_list[forward_begin +
|
||||||
const unsigned reverse_end =
|
leaf_object.fwd_segment_position - 1]
|
||||||
m_geometry_indices.at(leaf_object.reverse_packed_geometry_id + 1);
|
.node_id]);
|
||||||
|
v = &(internal_to_external_node_map
|
||||||
|
[m_geometry_list[forward_begin + leaf_object.fwd_segment_position]
|
||||||
|
.node_id]);
|
||||||
|
}
|
||||||
|
const double segment_length = util::coordinate_calculation::greatCircleDistance(
|
||||||
|
util::Coordinate{u->lon, u->lat}, util::Coordinate{v->lon, v->lat});
|
||||||
|
|
||||||
int rev_segment_position =
|
auto forward_speed_iter =
|
||||||
(reverse_end - reverse_begin) - leaf_object.fwd_segment_position - 1;
|
segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id));
|
||||||
if (rev_segment_position == 0)
|
if (forward_speed_iter != segment_speed_lookup.end())
|
||||||
{
|
{
|
||||||
u = &(internal_to_external_node_map[leaf_object.v]);
|
int new_segment_weight = std::max(
|
||||||
v = &(internal_to_external_node_map[m_geometry_list[reverse_begin]
|
1,
|
||||||
.node_id]);
|
static_cast<int>(std::floor(
|
||||||
}
|
(segment_length * 10.) / (forward_speed_iter->second.first / 3.6) +
|
||||||
else
|
.5)));
|
||||||
{
|
m_geometry_list[forward_begin + leaf_object.fwd_segment_position].weight =
|
||||||
u = &(internal_to_external_node_map
|
new_segment_weight;
|
||||||
[m_geometry_list[reverse_begin + rev_segment_position - 1]
|
m_geometry_datasource[forward_begin + leaf_object.fwd_segment_position] =
|
||||||
.node_id]);
|
forward_speed_iter->second.second;
|
||||||
v = &(
|
|
||||||
internal_to_external_node_map[m_geometry_list[reverse_begin +
|
|
||||||
rev_segment_position]
|
|
||||||
.node_id]);
|
|
||||||
}
|
|
||||||
const double segment_length =
|
|
||||||
util::coordinate_calculation::greatCircleDistance(
|
|
||||||
util::Coordinate{u->lon, u->lat}, util::Coordinate{v->lon, v->lat});
|
|
||||||
|
|
||||||
auto reverse_speed_iter =
|
|
||||||
segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id));
|
|
||||||
if (reverse_speed_iter != segment_speed_lookup.end())
|
|
||||||
{
|
|
||||||
int new_segment_weight =
|
|
||||||
std::max(1, static_cast<int>(std::floor(
|
|
||||||
(segment_length * 10.) /
|
|
||||||
(reverse_speed_iter->second.first / 3.6) +
|
|
||||||
.5)));
|
|
||||||
m_geometry_list[reverse_begin + rev_segment_position].weight =
|
|
||||||
new_segment_weight;
|
|
||||||
m_geometry_datasource[reverse_begin + rev_segment_position] =
|
|
||||||
reverse_speed_iter->second.second;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
--leaf_nodes_count;
|
if (leaf_object.reverse_packed_geometry_id != SPECIAL_EDGEID)
|
||||||
}
|
{
|
||||||
}
|
const unsigned reverse_begin =
|
||||||
|
m_geometry_indices.at(leaf_object.reverse_packed_geometry_id);
|
||||||
|
const unsigned reverse_end =
|
||||||
|
m_geometry_indices.at(leaf_object.reverse_packed_geometry_id + 1);
|
||||||
|
|
||||||
// Now save out the updated compressed geometries
|
int rev_segment_position =
|
||||||
{
|
(reverse_end - reverse_begin) - leaf_object.fwd_segment_position - 1;
|
||||||
std::ofstream geometry_stream(geometry_filename, std::ios::binary);
|
if (rev_segment_position == 0)
|
||||||
if (!geometry_stream)
|
{
|
||||||
{
|
u = &(internal_to_external_node_map[leaf_object.v]);
|
||||||
throw util::exception("Failed to open " + geometry_filename + " for writing");
|
v = &(
|
||||||
|
internal_to_external_node_map[m_geometry_list[reverse_begin].node_id]);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
u = &(
|
||||||
|
internal_to_external_node_map[m_geometry_list[reverse_begin +
|
||||||
|
rev_segment_position - 1]
|
||||||
|
.node_id]);
|
||||||
|
v = &(internal_to_external_node_map
|
||||||
|
[m_geometry_list[reverse_begin + rev_segment_position].node_id]);
|
||||||
|
}
|
||||||
|
const double segment_length = util::coordinate_calculation::greatCircleDistance(
|
||||||
|
util::Coordinate{u->lon, u->lat}, util::Coordinate{v->lon, v->lat});
|
||||||
|
|
||||||
|
auto reverse_speed_iter =
|
||||||
|
segment_speed_lookup.find(std::make_pair(u->node_id, v->node_id));
|
||||||
|
if (reverse_speed_iter != segment_speed_lookup.end())
|
||||||
|
{
|
||||||
|
int new_segment_weight = std::max(
|
||||||
|
1,
|
||||||
|
static_cast<int>(std::floor(
|
||||||
|
(segment_length * 10.) / (reverse_speed_iter->second.first / 3.6) +
|
||||||
|
.5)));
|
||||||
|
m_geometry_list[reverse_begin + rev_segment_position].weight =
|
||||||
|
new_segment_weight;
|
||||||
|
m_geometry_datasource[reverse_begin + rev_segment_position] =
|
||||||
|
reverse_speed_iter->second.second;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
const unsigned number_of_indices = m_geometry_indices.size();
|
--leaf_nodes_count;
|
||||||
const unsigned number_of_compressed_geometries = m_geometry_list.size();
|
|
||||||
geometry_stream.write(reinterpret_cast<const char *>(&number_of_indices),
|
|
||||||
sizeof(unsigned));
|
|
||||||
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_indices[0])),
|
|
||||||
number_of_indices * sizeof(unsigned));
|
|
||||||
geometry_stream.write(reinterpret_cast<const char *>(&number_of_compressed_geometries),
|
|
||||||
sizeof(unsigned));
|
|
||||||
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_list[0])),
|
|
||||||
number_of_compressed_geometries *
|
|
||||||
sizeof(extractor::CompressedEdgeContainer::CompressedEdge));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
const auto maybe_save_geometries = [&] {
|
||||||
|
if (!(update_edge_weights || update_turn_penalties))
|
||||||
|
return;
|
||||||
|
|
||||||
|
// Now save out the updated compressed geometries
|
||||||
|
std::ofstream geometry_stream(geometry_filename, std::ios::binary);
|
||||||
|
if (!geometry_stream)
|
||||||
|
{
|
||||||
|
throw util::exception("Failed to open " + geometry_filename + " for writing");
|
||||||
|
}
|
||||||
|
const unsigned number_of_indices = m_geometry_indices.size();
|
||||||
|
const unsigned number_of_compressed_geometries = m_geometry_list.size();
|
||||||
|
geometry_stream.write(reinterpret_cast<const char *>(&number_of_indices), sizeof(unsigned));
|
||||||
|
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_indices[0])),
|
||||||
|
number_of_indices * sizeof(unsigned));
|
||||||
|
geometry_stream.write(reinterpret_cast<const char *>(&number_of_compressed_geometries),
|
||||||
|
sizeof(unsigned));
|
||||||
|
geometry_stream.write(reinterpret_cast<char *>(&(m_geometry_list[0])),
|
||||||
|
number_of_compressed_geometries *
|
||||||
|
sizeof(extractor::CompressedEdgeContainer::CompressedEdge));
|
||||||
|
};
|
||||||
|
|
||||||
|
const auto save_datasource_indexes = [&] {
|
||||||
std::ofstream datasource_stream(datasource_indexes_filename, std::ios::binary);
|
std::ofstream datasource_stream(datasource_indexes_filename, std::ios::binary);
|
||||||
if (!datasource_stream)
|
if (!datasource_stream)
|
||||||
{
|
{
|
||||||
@ -511,9 +505,9 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
|
|||||||
datasource_stream.write(reinterpret_cast<char *>(&(m_geometry_datasource[0])),
|
datasource_stream.write(reinterpret_cast<char *>(&(m_geometry_datasource[0])),
|
||||||
number_of_datasource_entries * sizeof(uint8_t));
|
number_of_datasource_entries * sizeof(uint8_t));
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
|
|
||||||
{
|
const auto save_datastore_names = [&] {
|
||||||
std::ofstream datasource_stream(datasource_names_filename, std::ios::binary);
|
std::ofstream datasource_stream(datasource_names_filename, std::ios::binary);
|
||||||
if (!datasource_stream)
|
if (!datasource_stream)
|
||||||
{
|
{
|
||||||
@ -524,7 +518,9 @@ std::size_t Contractor::LoadEdgeExpandedGraph(
|
|||||||
{
|
{
|
||||||
datasource_stream << name << std::endl;
|
datasource_stream << name << std::endl;
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
|
|
||||||
|
tbb::parallel_invoke(maybe_save_geometries, save_datasource_indexes, save_datastore_names);
|
||||||
|
|
||||||
// TODO: can we read this in bulk? util::DeallocatingVector isn't necessarily
|
// TODO: can we read this in bulk? util::DeallocatingVector isn't necessarily
|
||||||
// all stored contiguously
|
// all stored contiguously
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user