From 498370330d8b0de60a596068c5bcc9bee08c5f74 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Wed, 4 May 2016 17:20:07 +0200 Subject: [PATCH] String bulk insertion --- src/extractor/extractor_callbacks.cpp | 61 ++++++++++++++------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/src/extractor/extractor_callbacks.cpp b/src/extractor/extractor_callbacks.cpp index 2fdf92362..dd2be66e7 100644 --- a/src/extractor/extractor_callbacks.cpp +++ b/src/extractor/extractor_callbacks.cpp @@ -4,8 +4,8 @@ #include "extractor/external_memory_node.hpp" #include "extractor/restriction.hpp" -#include "util/simple_logger.hpp" #include "util/for_each_pair.hpp" +#include "util/simple_logger.hpp" #include #include "extractor/extractor_callbacks.hpp" @@ -137,14 +137,23 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti road_classification.road_class = guidance::functionalRoadClassFromTag(data); } + // Deduplicates street names and street destination names based on the street_map map. + // In case we do not already store the name, inserts (name, id) tuple and return id. + // Otherwise fetches the id based on the name and returns it without insertion. + + const constexpr auto MAX_STRING_LENGTH = 255u; + // Get the unique identifier for the street name - const auto &string_map_iterator = string_map.find(parsed_way.name); + const auto string_map_iterator = string_map.find(parsed_way.name); unsigned name_id = external_memory.name_lengths.size(); if (string_map.end() == string_map_iterator) { - auto name_length = std::min(255u, parsed_way.name.size()); + auto name_length = std::min(MAX_STRING_LENGTH, parsed_way.name.size()); + + external_memory.name_char_data.reserve(name_id + name_length); std::copy(parsed_way.name.c_str(), parsed_way.name.c_str() + name_length, std::back_inserter(external_memory.name_char_data)); + external_memory.name_lengths.push_back(name_length); string_map.insert(std::make_pair(parsed_way.name, name_id)); } @@ -162,10 +171,7 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti std::transform(input_way.nodes().begin(), input_way.nodes().end(), std::back_inserter(external_memory.used_node_id_list), - [](const osmium::NodeRef &ref) - { - return OSMNodeID(ref.ref()); - }); + [](const osmium::NodeRef &ref) { return OSMNodeID(ref.ref()); }); const bool is_opposite_way = TRAVEL_MODE_INACCESSIBLE == parsed_way.forward_travel_mode; @@ -174,16 +180,15 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti { BOOST_ASSERT(split_edge == false); BOOST_ASSERT(parsed_way.backward_travel_mode != TRAVEL_MODE_INACCESSIBLE); - util::for_each_pair(input_way.nodes().crbegin(), input_way.nodes().crend(), - [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) - { - external_memory.all_edges_list.push_back(InternalExtractorEdge( - OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), - name_id, backward_weight_data, true, false, - parsed_way.roundabout, parsed_way.is_access_restricted, - parsed_way.is_startpoint, parsed_way.backward_travel_mode, - false, road_classification)); - }); + util::for_each_pair( + input_way.nodes().crbegin(), input_way.nodes().crend(), + [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) { + external_memory.all_edges_list.push_back(InternalExtractorEdge( + OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), name_id, + backward_weight_data, true, false, parsed_way.roundabout, + parsed_way.is_access_restricted, parsed_way.is_startpoint, + parsed_way.backward_travel_mode, false, road_classification)); + }); external_memory.way_start_end_id_list.push_back( {OSMWayID(input_way.id()), OSMNodeID(input_way.nodes().back().ref()), @@ -194,23 +199,21 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti { const bool forward_only = split_edge || TRAVEL_MODE_INACCESSIBLE == parsed_way.backward_travel_mode; - util::for_each_pair(input_way.nodes().cbegin(), input_way.nodes().cend(), - [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) - { - external_memory.all_edges_list.push_back(InternalExtractorEdge( - OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), - name_id, forward_weight_data, true, !forward_only, - parsed_way.roundabout, parsed_way.is_access_restricted, - parsed_way.is_startpoint, parsed_way.forward_travel_mode, - split_edge, road_classification)); - }); + util::for_each_pair( + input_way.nodes().cbegin(), input_way.nodes().cend(), + [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) { + external_memory.all_edges_list.push_back(InternalExtractorEdge( + OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), name_id, + forward_weight_data, true, !forward_only, parsed_way.roundabout, + parsed_way.is_access_restricted, parsed_way.is_startpoint, + parsed_way.forward_travel_mode, split_edge, road_classification)); + }); if (split_edge) { BOOST_ASSERT(parsed_way.backward_travel_mode != TRAVEL_MODE_INACCESSIBLE); util::for_each_pair( input_way.nodes().cbegin(), input_way.nodes().cend(), - [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) - { + [&](const osmium::NodeRef &first_node, const osmium::NodeRef &last_node) { external_memory.all_edges_list.push_back(InternalExtractorEdge( OSMNodeID(first_node.ref()), OSMNodeID(last_node.ref()), name_id, backward_weight_data, false, true, parsed_way.roundabout,