Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

First step into overhauling the edge storage

Browse Source
This commit is contained in:
Patrick Niklaus
2015-05-09 17:21:36 +02:00
parent c493a22765
commit a57fb4f1ab
16 changed files with 530 additions and 522 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extractor/extraction_containers.cpp
+361 -371
View File
@@ -54,6 +54,7 @@ ExtractionContainers::ExtractionContainers()
ExtractionContainers::~ExtractionContainers()
{
// FIXME isn't this done implicitly of the stxxl::vectors go out of scope?
used_node_id_list.clear();
all_nodes_list.clear();
all_edges_list.clear();
@@ -71,394 +72,383 @@ ExtractionContainers::~ExtractionContainers()
* - filter nodes list to nodes that are referenced by ways
* - merge edges with nodes to include location of start/end points and serialize
*
* FIXME: Each of this step should be an own function for readability.
*/
void ExtractionContainers::PrepareData(const std::string &output_file_name,
const std::string &restrictions_file_name)
const std::string &restrictions_file_name,
const std::string &name_file_name)
{
try
{
unsigned number_of_used_nodes = 0;
unsigned number_of_used_edges = 0;
std::cout << "[extractor] Sorting used nodes ... " << std::flush;
TIMER_START(sorting_used_nodes);
stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory);
TIMER_STOP(sorting_used_nodes);
std::cout << "ok, after " << TIMER_SEC(sorting_used_nodes) << "s" << std::endl;
std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush;
TIMER_START(erasing_dups);
auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end());
used_node_id_list.resize(new_end - used_node_id_list.begin());
TIMER_STOP(erasing_dups);
std::cout << "ok, after " << TIMER_SEC(erasing_dups) << "s" << std::endl;
std::cout << "[extractor] Sorting all nodes ... " << std::flush;
TIMER_START(sorting_nodes);
stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), ExternalMemoryNodeSTXXLCompare(),
stxxl_memory);
TIMER_STOP(sorting_nodes);
std::cout << "ok, after " << TIMER_SEC(sorting_nodes) << "s" << std::endl;
std::cout << "[extractor] Sorting used ways ... " << std::flush;
TIMER_START(sort_ways);
stxxl::sort(way_start_end_id_list.begin(), way_start_end_id_list.end(),
FirstAndLastSegmentOfWayStxxlCompare(), stxxl_memory);
TIMER_STOP(sort_ways);
std::cout << "ok, after " << TIMER_SEC(sort_ways) << "s" << std::endl;
std::cout << "[extractor] Sorting " << restrictions_list.size()
<< " restrictions. by from... " << std::flush;
TIMER_START(sort_restrictions);
stxxl::sort(restrictions_list.begin(), restrictions_list.end(),
CmpRestrictionContainerByFrom(), stxxl_memory);
TIMER_STOP(sort_restrictions);
std::cout << "ok, after " << TIMER_SEC(sort_restrictions) << "s" << std::endl;
std::cout << "[extractor] Fixing restriction starts ... " << std::flush;
TIMER_START(fix_restriction_starts);
auto restrictions_iterator = restrictions_list.begin();
auto way_start_and_end_iterator = way_start_end_id_list.cbegin();
while (way_start_and_end_iterator != way_start_end_id_list.cend() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.from.way)
{
++way_start_and_end_iterator;
continue;
}
if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.from.way)
{
++restrictions_iterator;
continue;
}
BOOST_ASSERT(way_start_and_end_iterator->way_id ==
restrictions_iterator->restriction.from.way);
const NodeID via_node_id = restrictions_iterator->restriction.via.node;
if (way_start_and_end_iterator->first_segment_source_id == via_node_id)
{
restrictions_iterator->restriction.from.node =
way_start_and_end_iterator->first_segment_target_id;
}
else if (way_start_and_end_iterator->last_segment_target_id == via_node_id)
{
restrictions_iterator->restriction.from.node =
way_start_and_end_iterator->last_segment_source_id;
}
++restrictions_iterator;
}
TIMER_STOP(fix_restriction_starts);
std::cout << "ok, after " << TIMER_SEC(fix_restriction_starts) << "s" << std::endl;
std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush;
TIMER_START(sort_restrictions_to);
stxxl::sort(restrictions_list.begin(), restrictions_list.end(),
CmpRestrictionContainerByTo(), stxxl_memory);
TIMER_STOP(sort_restrictions_to);
std::cout << "ok, after " << TIMER_SEC(sort_restrictions_to) << "s" << std::endl;
std::cout << "[extractor] Fixing restriction ends ... " << std::flush;
TIMER_START(fix_restriction_ends);
restrictions_iterator = restrictions_list.begin();
way_start_and_end_iterator = way_start_end_id_list.cbegin();
while (way_start_and_end_iterator != way_start_end_id_list.cend() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.to.way)
{
++way_start_and_end_iterator;
continue;
}
if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.to.way)
{
++restrictions_iterator;
continue;
}
BOOST_ASSERT(way_start_and_end_iterator->way_id ==
restrictions_iterator->restriction.to.way);
const NodeID via_node_id = restrictions_iterator->restriction.via.node;
if (way_start_and_end_iterator->first_segment_source_id == via_node_id)
{
restrictions_iterator->restriction.to.node =
way_start_and_end_iterator->first_segment_target_id;
}
else if (way_start_and_end_iterator->last_segment_target_id == via_node_id)
{
restrictions_iterator->restriction.to.node =
way_start_and_end_iterator->last_segment_source_id;
}
++restrictions_iterator;
}
TIMER_STOP(fix_restriction_ends);
std::cout << "ok, after " << TIMER_SEC(fix_restriction_ends) << "s" << std::endl;
// serialize restrictions
std::ofstream restrictions_out_stream;
unsigned written_restriction_count = 0;
restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary);
restrictions_out_stream.write((char *)&fingerprint, sizeof(FingerPrint));
const auto count_position = restrictions_out_stream.tellp();
restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned));
for (const auto &restriction_container : restrictions_list)
{
if (SPECIAL_NODEID != restriction_container.restriction.from.node &&
SPECIAL_NODEID != restriction_container.restriction.to.node)
{
restrictions_out_stream.write((char *)&(restriction_container.restriction),
sizeof(TurnRestriction));
++written_restriction_count;
}
}
restrictions_out_stream.seekp(count_position);
restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned));
restrictions_out_stream.close();
SimpleLogger().Write() << "usable restrictions: " << written_restriction_count;
PrepareRestrictions();
WriteRestrictions(restrictions_file_name);
std::ofstream file_out_stream;
file_out_stream.open(output_file_name.c_str(), std::ios::binary);
file_out_stream.write((char *)&fingerprint, sizeof(FingerPrint));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush;
TIMER_START(write_nodes);
// identify all used nodes by a merging step of two sorted lists
auto node_iterator = all_nodes_list.begin();
auto node_id_iterator = used_node_id_list.begin();
while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end())
{
if (*node_id_iterator < node_iterator->node_id)
{
++node_id_iterator;
continue;
}
if (*node_id_iterator > node_iterator->node_id)
{
++node_iterator;
continue;
}
BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode));
PrepareNodes();
WriteNodes(file_out_stream);
PrepareEdges();
WriteEdges(file_out_stream);
++number_of_used_nodes;
++node_id_iterator;
++node_iterator;
}
TIMER_STOP(write_nodes);
std::cout << "ok, after " << TIMER_SEC(write_nodes) << "s" << std::endl;
std::cout << "[extractor] setting number of nodes ... " << std::flush;
std::ios::pos_type previous_file_position = file_out_stream.tellp();
file_out_stream.seekp(std::ios::beg + sizeof(FingerPrint));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
file_out_stream.seekp(previous_file_position);
std::cout << "ok" << std::endl;
// Sort edges by start.
std::cout << "[extractor] Sorting edges by start ... " << std::flush;
TIMER_START(sort_edges_by_start);
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), stxxl_memory);
TIMER_STOP(sort_edges_by_start);
std::cout << "ok, after " << TIMER_SEC(sort_edges_by_start) << "s" << std::endl;
std::cout << "[extractor] Setting start coords ... " << std::flush;
TIMER_START(set_start_coords);
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord
node_iterator = all_nodes_list.begin();
auto edge_iterator = all_edges_list.begin();
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->start < node_iterator->node_id)
{
++edge_iterator;
continue;
}
if (edge_iterator->start > node_iterator->node_id)
{
node_iterator++;
continue;
}
BOOST_ASSERT(edge_iterator->start == node_iterator->node_id);
edge_iterator->source_coordinate.lat = node_iterator->lat;
edge_iterator->source_coordinate.lon = node_iterator->lon;
++edge_iterator;
}
TIMER_STOP(set_start_coords);
std::cout << "ok, after " << TIMER_SEC(set_start_coords) << "s" << std::endl;
// Sort Edges by target
std::cout << "[extractor] Sorting edges by target ... " << std::flush;
TIMER_START(sort_edges_by_target);
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(),
stxxl_memory);
TIMER_STOP(sort_edges_by_target);
std::cout << "ok, after " << TIMER_SEC(sort_edges_by_target) << "s" << std::endl;
std::cout << "[extractor] Setting target coords ... " << std::flush;
TIMER_START(set_target_coords);
// Traverse list of edges and nodes in parallel and set target coord
node_iterator = all_nodes_list.begin();
edge_iterator = all_edges_list.begin();
// Also serializes the edges
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->target < node_iterator->node_id)
{
++edge_iterator;
continue;
}
if (edge_iterator->target > node_iterator->node_id)
{
++node_iterator;
continue;
}
BOOST_ASSERT(edge_iterator->target == node_iterator->node_id);
if (edge_iterator->source_coordinate.lat != std::numeric_limits<int>::min() &&
edge_iterator->source_coordinate.lon != std::numeric_limits<int>::min())
{
BOOST_ASSERT(edge_iterator->speed != -1);
edge_iterator->target_coordinate.lat = node_iterator->lat;
edge_iterator->target_coordinate.lon = node_iterator->lon;
const double distance = coordinate_calculation::euclidean_distance(
edge_iterator->source_coordinate.lat, edge_iterator->source_coordinate.lon,
node_iterator->lat, node_iterator->lon);
const double weight = (distance * 10.) / (edge_iterator->speed / 3.6);
int integer_weight = std::max(
1, (int)std::floor(
(edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5));
// FIXME: This means we have a _minimum_ edge length of 1m
// maybe use dm as base unit?
const int integer_distance = std::max(1, (int)distance);
const short zero = 0;
const short one = 1;
const bool yes = true;
const bool no = false;
file_out_stream.write((char *)&edge_iterator->start, sizeof(unsigned));
file_out_stream.write((char *)&edge_iterator->target, sizeof(unsigned));
file_out_stream.write((char *)&integer_distance, sizeof(int));
switch (edge_iterator->direction)
{
case ExtractionWay::notSure:
file_out_stream.write((char *)&zero, sizeof(short));
break;
case ExtractionWay::oneway:
file_out_stream.write((char *)&one, sizeof(short));
break;
case ExtractionWay::bidirectional:
file_out_stream.write((char *)&zero, sizeof(short));
break;
case ExtractionWay::opposite:
file_out_stream.write((char *)&one, sizeof(short));
break;
default:
throw osrm::exception("edge has broken direction");
}
file_out_stream.write((char *)&integer_weight, sizeof(int));
file_out_stream.write((char *)&edge_iterator->name_id, sizeof(unsigned));
if (edge_iterator->is_roundabout)
{
file_out_stream.write((char *)&yes, sizeof(bool));
}
else
{
file_out_stream.write((char *)&no, sizeof(bool));
}
if (edge_iterator->is_in_tiny_cc)
{
file_out_stream.write((char *)&yes, sizeof(bool));
}
else
{
file_out_stream.write((char *)&no, sizeof(bool));
}
if (edge_iterator->is_access_restricted)
{
file_out_stream.write((char *)&yes, sizeof(bool));
}
else
{
file_out_stream.write((char *)&no, sizeof(bool));
}
// cannot take adress of bit field, so use local
const TravelMode travel_mode = edge_iterator->travel_mode;
file_out_stream.write((char *)&travel_mode, sizeof(TravelMode));
if (edge_iterator->is_split)
{
file_out_stream.write((char *)&yes, sizeof(bool));
}
else
{
file_out_stream.write((char *)&no, sizeof(bool));
}
++number_of_used_edges;
}
++edge_iterator;
}
TIMER_STOP(set_target_coords);
std::cout << "ok, after " << TIMER_SEC(set_target_coords) << "s" << std::endl;
std::cout << "[extractor] setting number of edges ... " << std::flush;
file_out_stream.seekp(previous_file_position);
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
file_out_stream.close();
std::cout << "ok" << std::endl;
std::cout << "[extractor] writing street name index ... " << std::flush;
TIMER_START(write_name_index);
std::string name_file_streamName = (output_file_name + ".names");
boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary);
unsigned total_length = 0;
std::vector<unsigned> name_lengths;
for (const std::string &temp_string : name_list)
{
const unsigned string_length =
std::min(static_cast<unsigned>(temp_string.length()), 255u);
name_lengths.push_back(string_length);
total_length += string_length;
}
RangeTable<> table(name_lengths);
name_file_stream << table;
name_file_stream.write((char *)&total_length, sizeof(unsigned));
// write all chars consecutively
for (const std::string &temp_string : name_list)
{
const unsigned string_length =
std::min(static_cast<unsigned>(temp_string.length()), 255u);
name_file_stream.write(temp_string.c_str(), string_length);
}
name_file_stream.close();
TIMER_STOP(write_name_index);
std::cout << "ok, after " << TIMER_SEC(write_name_index) << "s" << std::endl;
SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes and "
<< number_of_used_edges << " edges";
WriteNames(name_file_name);
}
catch (const std::exception &e)
{
std::cerr << "Caught Execption:" << e.what() << std::endl;
}
}
void ExtractionContainers::WriteNames(const std::string& names_file_name) const
{
std::cout << "[extractor] writing street name index ... " << std::flush;
TIMER_START(write_name_index);
boost::filesystem::ofstream name_file_stream(names_file_name, std::ios::binary);
unsigned total_length = 0;
std::vector<unsigned> name_lengths;
for (const std::string &temp_string : name_list)
{
const unsigned string_length =
std::min(static_cast<unsigned>(temp_string.length()), 255u);
name_lengths.push_back(string_length);
total_length += string_length;
}
// builds and writes the index
RangeTable<> name_index_range(name_lengths);
name_file_stream << name_index_range;
name_file_stream.write((char *)&total_length, sizeof(unsigned));
// write all chars consecutively
for (const std::string &temp_string : name_list)
{
const unsigned string_length =
std::min(static_cast<unsigned>(temp_string.length()), 255u);
name_file_stream.write(temp_string.c_str(), string_length);
}
name_file_stream.close();
TIMER_STOP(write_name_index);
std::cout << "ok, after " << TIMER_SEC(write_name_index) << "s" << std::endl;
}
void ExtractionContainers::PrepareNodes()
{
std::cout << "[extractor] Sorting used nodes ... " << std::flush;
TIMER_START(sorting_used_nodes);
stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory);
TIMER_STOP(sorting_used_nodes);
std::cout << "ok, after " << TIMER_SEC(sorting_used_nodes) << "s" << std::endl;
std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush;
TIMER_START(erasing_dups);
auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end());
used_node_id_list.resize(new_end - used_node_id_list.begin());
TIMER_STOP(erasing_dups);
std::cout << "ok, after " << TIMER_SEC(erasing_dups) << "s" << std::endl;
std::cout << "[extractor] Sorting all nodes ... " << std::flush;
TIMER_START(sorting_nodes);
stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), ExternalMemoryNodeSTXXLCompare(),
stxxl_memory);
TIMER_STOP(sorting_nodes);
std::cout << "ok, after " << TIMER_SEC(sorting_nodes) << "s" << std::endl;
}
void ExtractionContainers::PrepareEdges()
{
// Sort edges by start.
std::cout << "[extractor] Sorting edges by start ... " << std::flush;
TIMER_START(sort_edges_by_start);
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), stxxl_memory);
TIMER_STOP(sort_edges_by_start);
std::cout << "ok, after " << TIMER_SEC(sort_edges_by_start) << "s" << std::endl;
std::cout << "[extractor] Setting start coords ... " << std::flush;
TIMER_START(set_start_coords);
// Traverse list of edges and nodes in parallel and set start coord
auto node_iterator = all_nodes_list.begin();
auto edge_iterator = all_edges_list.begin();
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->result.source < node_iterator->node_id)
{
++edge_iterator;
continue;
}
if (edge_iterator->result.source > node_iterator->node_id)
{
node_iterator++;
continue;
}
BOOST_ASSERT(edge_iterator->result.source == node_iterator->node_id);
edge_iterator->source_coordinate.lat = node_iterator->lat;
edge_iterator->source_coordinate.lon = node_iterator->lon;
++edge_iterator;
}
TIMER_STOP(set_start_coords);
std::cout << "ok, after " << TIMER_SEC(set_start_coords) << "s" << std::endl;
// Sort Edges by target
std::cout << "[extractor] Sorting edges by target ... " << std::flush;
TIMER_START(sort_edges_by_target);
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(),
stxxl_memory);
TIMER_STOP(sort_edges_by_target);
std::cout << "ok, after " << TIMER_SEC(sort_edges_by_target) << "s" << std::endl;
// Compute edge weights
std::cout << "[extractor] Computing edge weights ... " << std::flush;
TIMER_START(compute_weights);
node_iterator = all_nodes_list.begin();
edge_iterator = all_edges_list.begin();
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->result.target < node_iterator->node_id)
{
// FIXME we are skipping edges here: That means the data is broken!
++edge_iterator;
continue;
}
if (edge_iterator->result.target > node_iterator->node_id)
{
++node_iterator;
continue;
}
BOOST_ASSERT(edge_iterator->result.target == node_iterator->node_id);
if (edge_iterator->source_coordinate.lat != std::numeric_limits<int>::min() &&
edge_iterator->source_coordinate.lon != std::numeric_limits<int>::min())
{
BOOST_ASSERT(edge_iterator->weight_data.speed != -1);
const double distance = coordinate_calculation::euclidean_distance(
edge_iterator->source_coordinate.lat, edge_iterator->source_coordinate.lon,
node_iterator->lat, node_iterator->lon);
const double weight = [distance](const InternalExtractorEdge::WeightData& data) {
switch (data.type)
{
case InternalExtractorEdge::WeightType::EDGE_DURATION:
case InternalExtractorEdge::WeightType::WAY_DURATION:
return data.duration * 10.;
break;
case InternalExtractorEdge::WeightType::SPEED:
return (distance * 10.) / (data.speed / 3.6);
break;
default:
osrm::exception("invalid weight type");
}
return -1.0;
}(edge_iterator->weight_data);
edge_iterator->result.weight = std::max(1, (int)std::floor(weight + .5));
}
++edge_iterator;
}
TIMER_STOP(compute_weights);
std::cout << "ok, after " << TIMER_SEC(compute_weights) << "s" << std::endl;
}
void ExtractionContainers::WriteEdges(std::ofstream& file_out_stream) const
{
std::cout << "[extractor] Writing used egdes ... " << std::flush;
TIMER_START(write_edges);
// Traverse list of edges and nodes in parallel and set target coord
unsigned number_of_used_edges = 0;
auto start_position = file_out_stream.tellp();
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
for (const auto& edge : all_edges_list)
{
if (edge.result.weight > 0)
{
file_out_stream.write((char*) &edge.result, sizeof(NodeBasedEdge));
number_of_used_edges++;
}
}
TIMER_STOP(write_edges);
std::cout << "ok, after " << TIMER_SEC(write_edges) << "s" << std::endl;
std::cout << "[extractor] setting number of edges ... " << std::flush;
file_out_stream.seekp(start_position);
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
std::cout << "ok" << std::endl;
SimpleLogger().Write() << "Processed " << number_of_used_edges << " nodes";
}
void ExtractionContainers::WriteNodes(std::ofstream& file_out_stream) const
{
unsigned number_of_used_nodes = 0;
// write dummy value, will be overwritten later
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush;
TIMER_START(write_nodes);
// identify all used nodes by a merging step of two sorted lists
auto node_iterator = all_nodes_list.begin();
auto node_id_iterator = used_node_id_list.begin();
while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end())
{
if (*node_id_iterator < node_iterator->node_id)
{
++node_id_iterator;
continue;
}
if (*node_id_iterator > node_iterator->node_id)
{
++node_iterator;
continue;
}
BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode));
++number_of_used_nodes;
++node_id_iterator;
++node_iterator;
}
TIMER_STOP(write_nodes);
std::cout << "ok, after " << TIMER_SEC(write_nodes) << "s" << std::endl;
std::cout << "[extractor] setting number of nodes ... " << std::flush;
std::ios::pos_type previous_file_position = file_out_stream.tellp();
file_out_stream.seekp(std::ios::beg + sizeof(FingerPrint));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
file_out_stream.seekp(previous_file_position);
std::cout << "ok" << std::endl;
SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes";
}
void ExtractionContainers::WriteRestrictions(const std::string& path) const
{
// serialize restrictions
std::ofstream restrictions_out_stream;
unsigned written_restriction_count = 0;
restrictions_out_stream.open(path.c_str(), std::ios::binary);
restrictions_out_stream.write((char *)&fingerprint, sizeof(FingerPrint));
const auto count_position = restrictions_out_stream.tellp();
restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned));
for (const auto &restriction_container : restrictions_list)
{
if (SPECIAL_NODEID != restriction_container.restriction.from.node &&
SPECIAL_NODEID != restriction_container.restriction.to.node)
{
restrictions_out_stream.write((char *)&(restriction_container.restriction),
sizeof(TurnRestriction));
++written_restriction_count;
}
}
restrictions_out_stream.seekp(count_position);
restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned));
restrictions_out_stream.close();
SimpleLogger().Write() << "usable restrictions: " << written_restriction_count;
}
void ExtractionContainers::PrepareRestrictions()
{
std::cout << "[extractor] Sorting used ways ... " << std::flush;
TIMER_START(sort_ways);
stxxl::sort(way_start_end_id_list.begin(), way_start_end_id_list.end(),
FirstAndLastSegmentOfWayStxxlCompare(), stxxl_memory);
TIMER_STOP(sort_ways);
std::cout << "ok, after " << TIMER_SEC(sort_ways) << "s" << std::endl;
std::cout << "[extractor] Sorting " << restrictions_list.size()
<< " restrictions. by from... " << std::flush;
TIMER_START(sort_restrictions);
stxxl::sort(restrictions_list.begin(), restrictions_list.end(),
CmpRestrictionContainerByFrom(), stxxl_memory);
TIMER_STOP(sort_restrictions);
std::cout << "ok, after " << TIMER_SEC(sort_restrictions) << "s" << std::endl;
std::cout << "[extractor] Fixing restriction starts ... " << std::flush;
TIMER_START(fix_restriction_starts);
auto restrictions_iterator = restrictions_list.begin();
auto way_start_and_end_iterator = way_start_end_id_list.cbegin();
while (way_start_and_end_iterator != way_start_end_id_list.cend() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.from.way)
{
++way_start_and_end_iterator;
continue;
}
if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.from.way)
{
++restrictions_iterator;
continue;
}
BOOST_ASSERT(way_start_and_end_iterator->way_id ==
restrictions_iterator->restriction.from.way);
const NodeID via_node_id = restrictions_iterator->restriction.via.node;
if (way_start_and_end_iterator->first_segment_source_id == via_node_id)
{
restrictions_iterator->restriction.from.node =
way_start_and_end_iterator->first_segment_target_id;
}
else if (way_start_and_end_iterator->last_segment_target_id == via_node_id)
{
restrictions_iterator->restriction.from.node =
way_start_and_end_iterator->last_segment_source_id;
}
++restrictions_iterator;
}
TIMER_STOP(fix_restriction_starts);
std::cout << "ok, after " << TIMER_SEC(fix_restriction_starts) << "s" << std::endl;
std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush;
TIMER_START(sort_restrictions_to);
stxxl::sort(restrictions_list.begin(), restrictions_list.end(),
CmpRestrictionContainerByTo(), stxxl_memory);
TIMER_STOP(sort_restrictions_to);
std::cout << "ok, after " << TIMER_SEC(sort_restrictions_to) << "s" << std::endl;
std::cout << "[extractor] Fixing restriction ends ... " << std::flush;
TIMER_START(fix_restriction_ends);
restrictions_iterator = restrictions_list.begin();
way_start_and_end_iterator = way_start_end_id_list.cbegin();
while (way_start_and_end_iterator != way_start_end_id_list.cend() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.to.way)
{
++way_start_and_end_iterator;
continue;
}
if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.to.way)
{
++restrictions_iterator;
continue;
}
BOOST_ASSERT(way_start_and_end_iterator->way_id ==
restrictions_iterator->restriction.to.way);
const NodeID via_node_id = restrictions_iterator->restriction.via.node;
if (way_start_and_end_iterator->first_segment_source_id == via_node_id)
{
restrictions_iterator->restriction.to.node =
way_start_and_end_iterator->first_segment_target_id;
}
else if (way_start_and_end_iterator->last_segment_target_id == via_node_id)
{
restrictions_iterator->restriction.to.node =
way_start_and_end_iterator->last_segment_source_id;
}
++restrictions_iterator;
}
TIMER_STOP(fix_restriction_ends);
std::cout << "ok, after " << TIMER_SEC(fix_restriction_ends) << "s" << std::endl;
}
extractor/extraction_containers.hpp
+12 -1
View File
@@ -39,6 +39,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/**
* Uses external memory containers from stxxl to store all the data that
* is collected by the extractor callbacks.
*
* The data is the filtered, aggregated and finally written to disk.
*/
class ExtractionContainers
{
@@ -49,6 +51,14 @@ class ExtractionContainers
#else
const static unsigned stxxl_memory = ((sizeof(std::size_t) == 4) ? INT_MAX : UINT_MAX);
#endif
void PrepareNodes();
void PrepareRestrictions();
void PrepareEdges();
void WriteNodes(std::ofstream& file_out_stream) const;
void WriteRestrictions(const std::string& restrictions_file_name) const;
void WriteEdges(std::ofstream& file_out_stream) const;
void WriteNames(const std::string& names_file_name) const;
public:
using STXXLNodeIDVector = stxxl::vector<NodeID>;
using STXXLNodeVector = stxxl::vector<ExternalMemoryNode>;
@@ -70,7 +80,8 @@ class ExtractionContainers
~ExtractionContainers();
void PrepareData(const std::string &output_file_name,
const std::string &restrictions_file_name);
const std::string &restrictions_file_name,
const std::string &names_file_name);
};
#endif /* EXTRACTION_CONTAINERS_HPP */
extractor/extraction_helper_functions.hpp
+1 -1
View File
@@ -102,7 +102,7 @@ unsigned parseDuration(const std::string &s)
minutes = cast::string_to_int(result[1]);
hours = cast::string_to_int(result[0]);
}
return 10 * (3600 * hours + 60 * minutes + seconds);
return (3600 * hours + 60 * minutes + seconds);
}
}
else if (iso_8601_duration_is_valid(s))
extractor/extractor.cpp
+2 -1
View File
@@ -237,7 +237,8 @@ int extractor::run()
}
extraction_containers.PrepareData(config.output_file_name,
config.restriction_file_name);
config.restriction_file_name,
config.names_file_name);
TIMER_STOP(extracting);
SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s";
SimpleLogger().Write() << "To prepare the data for routing, run: "
extractor/extractor_callbacks.cpp
+46 -28
View File
@@ -108,17 +108,40 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti
<< " of size " << input_way.nodes().size();
return;
}
InternalExtractorEdge::WeightData forward_weight_data;
InternalExtractorEdge::WeightData backward_weight_data;
if (0 < parsed_way.duration)
{
// TODO: iterate all way segments and set duration corresponding to the length of each
// segment
const_cast<ExtractionWay &>(parsed_way).forward_speed =
parsed_way.duration / (input_way.nodes().size() - 1);
const_cast<ExtractionWay &>(parsed_way).backward_speed =
parsed_way.duration / (input_way.nodes().size() - 1);
const unsigned num_edges = (input_way.nodes().size() - 1);
// FIXME We devide by the numer of nodes here, but should rather consider
// the length of each segment. We would eigther have to compute the length
// of the whole way here (we can't: no node coordinates) or push that back
// to the container and keep a reference to the way.
forward_weight_data.duration = parsed_way.duration / num_edges;
forward_weight_data.type = InternalExtractorEdge::WeightType::WAY_DURATION;
backward_weight_data.duration = parsed_way.duration / num_edges;
backward_weight_data.type = InternalExtractorEdge::WeightType::WAY_DURATION;
}
else
{
if (parsed_way.forward_speed > 0 &&
parsed_way.forward_travel_mode != TRAVEL_MODE_INACCESSIBLE)
{
forward_weight_data.speed = parsed_way.forward_speed;
forward_weight_data.type = InternalExtractorEdge::WeightType::SPEED;
}
if (parsed_way.backward_speed > 0 &&
parsed_way.backward_travel_mode != TRAVEL_MODE_INACCESSIBLE)
{
backward_weight_data.speed = parsed_way.backward_speed;
backward_weight_data.type = InternalExtractorEdge::WeightType::SPEED;
}
}
if (std::numeric_limits<double>::epsilon() >= std::abs(-1. - parsed_way.forward_speed))
if (forward_weight_data.type == InternalExtractorEdge::WeightType::INVALID
&& backward_weight_data.type == InternalExtractorEdge::WeightType::INVALID)
{
SimpleLogger().Write(logDEBUG) << "found way with bogus speed, id: " << input_way.id();
return;
@@ -144,36 +167,34 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti
((parsed_way.forward_speed != parsed_way.backward_speed) ||
(parsed_way.forward_travel_mode != parsed_way.backward_travel_mode));
auto pair_wise_segment_split =
// lambda to add edge to container
auto pair_wise_segment_split_forward =
[&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node)
{
// SimpleLogger().Write() << "adding edge (" << first_node.ref() << "," <<
// last_node.ref() << "), fwd speed: " << parsed_way.forward_speed;
const bool forward_only = split_edge || TRAVEL_MODE_INACCESSIBLE == parsed_way.backward_travel_mode;
external_memory.all_edges_list.push_back(InternalExtractorEdge(
first_node.ref(), last_node.ref(),
((split_edge || TRAVEL_MODE_INACCESSIBLE == parsed_way.backward_travel_mode)
? ExtractionWay::oneway
: ExtractionWay::bidirectional),
parsed_way.forward_speed, name_id, parsed_way.roundabout, parsed_way.ignore_in_grid,
(0 < parsed_way.duration), parsed_way.is_access_restricted,
parsed_way.forward_travel_mode, split_edge));
first_node.ref(), last_node.ref(), name_id, forward_weight_data,
true, !forward_only, parsed_way.roundabout, parsed_way.ignore_in_grid,
parsed_way.is_access_restricted, parsed_way.forward_travel_mode, split_edge));
external_memory.used_node_id_list.push_back(first_node.ref());
};
const bool is_opposite_way = TRAVEL_MODE_INACCESSIBLE == parsed_way.forward_travel_mode;
// traverse way in reverse in this case
if (is_opposite_way)
{
// why don't we have to swap the parsed_way.forward/backward speed here as well
const_cast<ExtractionWay &>(parsed_way).forward_travel_mode =
parsed_way.backward_travel_mode;
const_cast<ExtractionWay &>(parsed_way).backward_travel_mode = TRAVEL_MODE_INACCESSIBLE;
osrm::for_each_pair(input_way.nodes().crbegin(), input_way.nodes().crend(),
pair_wise_segment_split);
pair_wise_segment_split_forward);
external_memory.used_node_id_list.push_back(input_way.nodes().front().ref());
}
else
{
osrm::for_each_pair(input_way.nodes().cbegin(), input_way.nodes().cend(),
pair_wise_segment_split);
pair_wise_segment_split_forward);
external_memory.used_node_id_list.push_back(input_way.nodes().back().ref());
}
@@ -187,22 +208,19 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti
if (split_edge)
{ // Only true if the way should be split
BOOST_ASSERT(parsed_way.backward_travel_mode > 0);
BOOST_ASSERT(parsed_way.backward_travel_mode != TRAVEL_MODE_INACCESSIBLE);
auto pair_wise_segment_split_2 =
[&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node)
{
// SimpleLogger().Write() << "adding edge (" << last_node.ref() << "," <<
// first_node.ref() << "), bwd speed: " << parsed_way.backward_speed;
external_memory.all_edges_list.push_back(InternalExtractorEdge(
last_node.ref(), first_node.ref(), ExtractionWay::oneway, parsed_way.backward_speed,
name_id, parsed_way.roundabout, parsed_way.ignore_in_grid,
(0 < parsed_way.duration), parsed_way.is_access_restricted,
parsed_way.backward_travel_mode, split_edge));
last_node.ref(), first_node.ref(), name_id, backward_weight_data,
true, false, parsed_way.roundabout, parsed_way.ignore_in_grid,
parsed_way.is_access_restricted, parsed_way.backward_travel_mode, split_edge));
external_memory.used_node_id_list.push_back(last_node.ref());
};
if (is_opposite_way)
{
// SimpleLogger().Write() << "opposite2";
osrm::for_each_pair(input_way.nodes().crbegin(), input_way.nodes().crend(),
pair_wise_segment_split_2);
external_memory.used_node_id_list.push_back(input_way.nodes().front().ref());
@@ -211,7 +229,7 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti
{
osrm::for_each_pair(input_way.nodes().cbegin(), input_way.nodes().cend(),
pair_wise_segment_split_2);
external_memory.used_node_id_list.push_back(input_way.nodes().back().ref());
external_memory.used_node_id_list.push_back(input_way.nodes().front().ref());
}
external_memory.way_start_end_id_list.push_back(
extractor/extractor_options.cpp
+4
View File
@@ -135,6 +135,7 @@ void ExtractorOptions::GenerateOutputFilesNames(ExtractorConfig &extractor_confi
boost::filesystem::path &input_path = extractor_config.input_path;
extractor_config.output_file_name = input_path.string();
extractor_config.restriction_file_name = input_path.string();
extractor_config.names_file_name = input_path.string();
extractor_config.timestamp_file_name = input_path.string();
std::string::size_type pos = extractor_config.output_file_name.find(".osm.bz2");
if (pos == std::string::npos)
@@ -156,12 +157,14 @@ void ExtractorOptions::GenerateOutputFilesNames(ExtractorConfig &extractor_confi
{
extractor_config.output_file_name.append(".osrm");
extractor_config.restriction_file_name.append(".osrm.restrictions");
extractor_config.names_file_name.append(".osrm.names");
extractor_config.timestamp_file_name.append(".osrm.timestamp");
}
else
{
extractor_config.output_file_name.replace(pos, 5, ".osrm");
extractor_config.restriction_file_name.replace(pos, 5, ".osrm.restrictions");
extractor_config.names_file_name.replace(pos, 5, ".osrm.names");
extractor_config.timestamp_file_name.replace(pos, 5, ".osrm.timestamp");
}
}
@@ -169,6 +172,7 @@ void ExtractorOptions::GenerateOutputFilesNames(ExtractorConfig &extractor_confi
{
extractor_config.output_file_name.replace(pos, 8, ".osrm");
extractor_config.restriction_file_name.replace(pos, 8, ".osrm.restrictions");
extractor_config.names_file_name.replace(pos, 8, ".osrm.names");
extractor_config.timestamp_file_name.replace(pos, 8, ".osrm.timestamp");
}
}
extractor/extractor_options.hpp
+1
View File
@@ -48,6 +48,7 @@ struct ExtractorConfig
std::string output_file_name;
std::string restriction_file_name;
std::string names_file_name;
std::string timestamp_file_name;
unsigned requested_num_threads;
extractor/internal_extractor_edge.hpp
+57 -45
View File
@@ -30,6 +30,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../typedefs.h"
#include "../data_structures/travel_mode.hpp"
#include "../data_structures/import_edge.hpp"
#include <boost/assert.hpp>
@@ -37,83 +38,94 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
struct InternalExtractorEdge
{
InternalExtractorEdge()
: start(0), target(0), speed(0), name_id(0), direction(0), is_roundabout(false),
is_in_tiny_cc(false), is_duration_set(false), is_access_restricted(false),
is_split(false), travel_mode(TRAVEL_MODE_INACCESSIBLE)
// specify the type of the weight data
enum class WeightType : char {
INVALID,
SPEED,
EDGE_DURATION,
WAY_DURATION,
};
struct WeightData
{
WeightData() : duration(0.0), type(WeightType::INVALID)
{
}
union
{
double duration;
double speed;
};
WeightType type;
};
explicit InternalExtractorEdge()
: result(0, 0, 0, 0, false, false, false, false, false,
TRAVEL_MODE_INACCESSIBLE, false)
{
}
explicit InternalExtractorEdge(NodeID start,
NodeID target,
short direction,
double speed,
unsigned name_id,
bool is_roundabout,
bool is_in_tiny_cc,
bool is_duration_set,
bool is_access_restricted,
TravelMode travel_mode,
bool is_split)
: start(start), target(target), speed(speed), name_id(name_id), direction(direction),
is_roundabout(is_roundabout), is_in_tiny_cc(is_in_tiny_cc),
is_duration_set(is_duration_set), is_access_restricted(is_access_restricted),
is_split(is_split), travel_mode(travel_mode)
explicit InternalExtractorEdge(NodeID source,
NodeID target,
NodeID name_id,
const WeightData& weight_data,
bool forward,
bool backward,
bool roundabout,
bool in_tiny_cc,
bool access_restricted,
TravelMode travel_mode,
bool is_split)
: result(source, target, name_id, 0, forward, backward, roundabout,
in_tiny_cc, access_restricted, travel_mode, is_split),
weight_data(weight_data)
{
}
// data that will be written to disk
NodeBasedEdge result;
// intermediate edge weight
WeightData weight_data;
// coordinate of the source node
FixedPointCoordinate source_coordinate;
// necessary static util functions for stxxl's sorting
static InternalExtractorEdge min_value()
{
return InternalExtractorEdge(0, 0, 0, 0, 0, false, false, false, false,
TRAVEL_MODE_INACCESSIBLE, false);
return InternalExtractorEdge(0, 0, 0, WeightData(), false, false, false,
false, false, TRAVEL_MODE_INACCESSIBLE, false);
}
static InternalExtractorEdge max_value()
{
return InternalExtractorEdge(SPECIAL_NODEID, SPECIAL_NODEID, 0, 0, 0, false, false, false,
false, TRAVEL_MODE_INACCESSIBLE, false);
return InternalExtractorEdge(SPECIAL_NODEID, SPECIAL_NODEID, 0, WeightData(), false, false,
false, false, false, TRAVEL_MODE_INACCESSIBLE, false);
}
NodeID start;
NodeID target;
double speed;
unsigned name_id;
short direction;
bool is_roundabout : 1;
bool is_in_tiny_cc : 1;
bool is_duration_set : 1;
bool is_access_restricted : 1;
bool is_split : 1;
TravelMode travel_mode : 4;
FixedPointCoordinate source_coordinate;
FixedPointCoordinate target_coordinate;
};
struct CmpEdgeByStartID
{
using value_type = InternalExtractorEdge;
bool operator()(const InternalExtractorEdge &a, const InternalExtractorEdge &b) const
bool operator()(const InternalExtractorEdge &lhs, const InternalExtractorEdge &rhs) const
{
return a.start < b.start;
return lhs.result.source < rhs.result.source;
}
value_type max_value() { return InternalExtractorEdge::max_value(); }
value_type min_value() { return InternalExtractorEdge::min_value(); }
};
struct CmpEdgeByTargetID
{
using value_type = InternalExtractorEdge;
bool operator()(const InternalExtractorEdge &a, const InternalExtractorEdge &b) const
bool operator()(const InternalExtractorEdge &lhs, const InternalExtractorEdge &rhs) const
{
return a.target < b.target;
return lhs.result.target < rhs.result.target;
}
value_type max_value() { return InternalExtractorEdge::max_value(); }
value_type min_value() { return InternalExtractorEdge::min_value(); }
};