Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

String bulk insertion

Browse Source
This commit is contained in:
Daniel J. Hofmann 2016-05-04 17:20:07 +02:00 committed by Patrick Niklaus
parent 6865853776
commit 498370330d
No known key found for this signature in database
GPG Key ID: E426891B5F978B1B
1 changed files with 32 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
src/extractor/extractor_callbacks.cpp
View File

@ -4,8 +4,8 @@
#include "extractor/external_memory_node.hpp" #include "extractor/external_memory_node.hpp"
#include "extractor/restriction.hpp" #include "extractor/restriction.hpp"
#include "util/simple_logger.hpp"
#include "util/for_each_pair.hpp" #include "util/for_each_pair.hpp"
#include "util/simple_logger.hpp"
#include <boost/optional/optional.hpp> #include <boost/optional/optional.hpp>
#include "extractor/extractor_callbacks.hpp" #include "extractor/extractor_callbacks.hpp"
@ -137,14 +137,23 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti
road_classification.road_class = guidance::functionalRoadClassFromTag(data); road_classification.road_class = guidance::functionalRoadClassFromTag(data);
} }
// Deduplicates street names and street destination names based on the street_map map.
// In case we do not already store the name, inserts (name, id) tuple and return id.
// Otherwise fetches the id based on the name and returns it without insertion.
const constexpr auto MAX_STRING_LENGTH = 255u;
// Get the unique identifier for the street name // Get the unique identifier for the street name
const auto &string_map_iterator = string_map.find(parsed_way.name); const auto string_map_iterator = string_map.find(parsed_way.name);
unsigned name_id = external_memory.name_lengths.size(); unsigned name_id = external_memory.name_lengths.size();
if (string_map.end() == string_map_iterator) if (string_map.end() == string_map_iterator)
{ {
auto name_length = std::min<unsigned>(255u, parsed_way.name.size()); auto name_length = std::min<unsigned>(MAX_STRING_LENGTH, parsed_way.name.size());
external_memory.name_char_data.reserve(name_id + name_length);
std::copy(parsed_way.name.c_str(), parsed_way.name.c_str() + name_length, std::copy(parsed_way.name.c_str(), parsed_way.name.c_str() + name_length,
std::back_inserter(external_memory.name_char_data)); std::back_inserter(external_memory.name_char_data));
external_memory.name_lengths.push_back(name_length); external_memory.name_lengths.push_back(name_length);
string_map.insert(std::make_pair(parsed_way.name, name_id)); string_map.insert(std::make_pair(parsed_way.name, name_id));
} }
@ -162,10 +171,7 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti
std::transform(input_way.nodes().begin(), input_way.nodes().end(), std::transform(input_way.nodes().begin(), input_way.nodes().end(),
std::back_inserter(external_memory.used_node_id_list), std::back_inserter(external_memory.used_node_id_list),
[](const osmium::NodeRef &ref) [](const osmium::NodeRef &ref) { return OSMNodeID(ref.ref()); });
{
return OSMNodeID(ref.ref());
});
const bool is_opposite_way = TRAVEL_MODE_INACCESSIBLE == parsed_way.forward_travel_mode; const bool is_opposite_way = TRAVEL_MODE_INACCESSIBLE == parsed_way.forward_travel_mode;
@ -174,15 +180,14 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti
{ {
BOOST_ASSERT(split_edge == false); BOOST_ASSERT(split_edge == false);
BOOST_ASSERT(parsed_way.backward_travel_mode != TRAVEL_MODE_INACCESSIBLE); BOOST_ASSERT(parsed_way.backward_travel_mode != TRAVEL_MODE_INACCESSIBLE);
util::for_each_pair(input_way.nodes().crbegin(), input_way.nodes().crend(), util::for_each_pair(
[&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) input_way.nodes().crbegin(), input_way.nodes().crend(),
{ [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) {
external_memory.all_edges_list.push_back(InternalExtractorEdge( external_memory.all_edges_list.push_back(InternalExtractorEdge(
OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), name_id,
name_id, backward_weight_data, true, false, backward_weight_data, true, false, parsed_way.roundabout,
parsed_way.roundabout, parsed_way.is_access_restricted, parsed_way.is_access_restricted, parsed_way.is_startpoint,
parsed_way.is_startpoint, parsed_way.backward_travel_mode, parsed_way.backward_travel_mode, false, road_classification));
false, road_classification));
}); });
external_memory.way_start_end_id_list.push_back( external_memory.way_start_end_id_list.push_back(
@ -194,23 +199,21 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti
{ {
const bool forward_only = const bool forward_only =
split_edge || TRAVEL_MODE_INACCESSIBLE == parsed_way.backward_travel_mode; split_edge || TRAVEL_MODE_INACCESSIBLE == parsed_way.backward_travel_mode;
util::for_each_pair(input_way.nodes().cbegin(), input_way.nodes().cend(), util::for_each_pair(
[&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) input_way.nodes().cbegin(), input_way.nodes().cend(),
{ [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) {
external_memory.all_edges_list.push_back(InternalExtractorEdge( external_memory.all_edges_list.push_back(InternalExtractorEdge(
OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), name_id,
name_id, forward_weight_data, true, !forward_only, forward_weight_data, true, !forward_only, parsed_way.roundabout,
parsed_way.roundabout, parsed_way.is_access_restricted, parsed_way.is_access_restricted, parsed_way.is_startpoint,
parsed_way.is_startpoint, parsed_way.forward_travel_mode, parsed_way.forward_travel_mode, split_edge, road_classification));
split_edge, road_classification));
}); });
if (split_edge) if (split_edge)
{ {
BOOST_ASSERT(parsed_way.backward_travel_mode != TRAVEL_MODE_INACCESSIBLE); BOOST_ASSERT(parsed_way.backward_travel_mode != TRAVEL_MODE_INACCESSIBLE);
util::for_each_pair( util::for_each_pair(
input_way.nodes().cbegin(), input_way.nodes().cend(), input_way.nodes().cbegin(), input_way.nodes().cend(),
[&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) {
{
external_memory.all_edges_list.push_back(InternalExtractorEdge( external_memory.all_edges_list.push_back(InternalExtractorEdge(
OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), name_id, OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), name_id,
backward_weight_data, false, true, parsed_way.roundabout, backward_weight_data, false, true, parsed_way.roundabout,