From 8e6d0183b7cc3209a3f9f3220138ffd665260854 Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Wed, 31 Aug 2022 23:28:47 +0200 Subject: [PATCH] Deprecate .osrm file --- include/extractor/extraction_containers.hpp | 3 - include/extractor/extractor_config.hpp | 1 + src/extractor/extraction_containers.cpp | 133 +++++++++----------- src/extractor/extractor.cpp | 11 +- src/tools/extract.cpp | 7 +- 5 files changed, 73 insertions(+), 82 deletions(-) diff --git a/include/extractor/extraction_containers.hpp b/include/extractor/extraction_containers.hpp index 0813b29b1..9aaa27c0d 100644 --- a/include/extractor/extraction_containers.hpp +++ b/include/extractor/extraction_containers.hpp @@ -43,8 +43,6 @@ class ExtractionContainers void PrepareTrafficSignals(const ReferencedTrafficSignals &referenced_traffic_signals); void PrepareEdges(ScriptingEnvironment &scripting_environment); - void WriteNodes(storage::tar::FileWriter &file_out); - void WriteEdges(storage::tar::FileWriter &file_out); void WriteCharData(const std::string &file_name); public: @@ -91,7 +89,6 @@ class ExtractionContainers ExtractionContainers(); void PrepareData(ScriptingEnvironment &scripting_environment, - const std::string &osrm_path, const std::string &names_data_path); }; } // namespace extractor diff --git a/include/extractor/extractor_config.hpp b/include/extractor/extractor_config.hpp index 848d97add..81740297a 100644 --- a/include/extractor/extractor_config.hpp +++ b/include/extractor/extractor_config.hpp @@ -89,6 +89,7 @@ struct ExtractorConfig final : storage::IOConfig bool use_metadata = false; bool parse_conditionals = false; bool use_locations_cache = true; + bool dump_nbg_graph = false; }; } // namespace extractor } // namespace osrm diff --git a/src/extractor/extraction_containers.cpp b/src/extractor/extraction_containers.cpp index 89dc202bc..d8b51f16e 100644 --- a/src/extractor/extraction_containers.cpp +++ b/src/extractor/extraction_containers.cpp @@ -407,21 +407,14 @@ ExtractionContainers::ExtractionContainers() * */ void ExtractionContainers::PrepareData(ScriptingEnvironment &scripting_environment, - const std::string &osrm_path, const std::string &name_file_name) { - storage::tar::FileWriter writer(osrm_path, storage::tar::FileWriter::GenerateFingerprint); - const auto restriction_ways = IdentifyRestrictionWays(); const auto maneuver_override_ways = IdentifyManeuverOverrideWays(); const auto traffic_signals = IdentifyTrafficSignals(); PrepareNodes(); - WriteNodes(writer); PrepareEdges(scripting_environment); - all_nodes_list.clear(); // free all_nodes_list before allocation of normal_edges - all_nodes_list.shrink_to_fit(); - WriteEdges(writer); PrepareTrafficSignals(traffic_signals); PrepareManeuverOverrides(maneuver_override_ways); @@ -518,6 +511,59 @@ void ExtractionContainers::PrepareNodes() TIMER_STOP(id_map); log << "ok, after " << TIMER_SEC(id_map) << "s"; } + { + util::UnbufferedLog log; + log << "Confirming/Writing used nodes ... "; + TIMER_START(write_nodes); + // identify all used nodes by a merging step of two sorted lists + auto node_iterator = all_nodes_list.begin(); + auto node_id_iterator = used_node_id_list.begin(); + const auto all_nodes_list_end = all_nodes_list.end(); + + for (size_t index = 0; index < used_node_id_list.size(); ++index) + { + BOOST_ASSERT(node_id_iterator != used_node_id_list.end()); + BOOST_ASSERT(node_iterator != all_nodes_list_end); + BOOST_ASSERT(*node_id_iterator >= node_iterator->node_id); + while (*node_id_iterator > node_iterator->node_id && + node_iterator != all_nodes_list_end) + { + ++node_iterator; + } + if (node_iterator == all_nodes_list_end || *node_id_iterator < node_iterator->node_id) + { + throw util::exception( + "Invalid OSM data: Referenced non-existing node with ID " + + std::to_string(static_cast(*node_id_iterator))); + } + BOOST_ASSERT(*node_id_iterator == node_iterator->node_id); + + ++node_id_iterator; + + internal_nodes.emplace_back(*node_iterator++); + } + + TIMER_STOP(write_nodes); + log << "ok, after " << TIMER_SEC(write_nodes) << "s"; + } + + { + util::UnbufferedLog log; + log << "Writing barrier nodes ... "; + TIMER_START(write_nodes); + for (const auto osm_id : barrier_nodes) + { + const auto node_id = mapExternalToInternalNodeID( + used_node_id_list.begin(), used_node_id_list.end(), osm_id); + if (node_id != SPECIAL_NODEID) + { + internal_barrier_nodes.emplace(node_id); + } + } + log << "ok, after " << TIMER_SEC(write_nodes) << "s"; + } + + util::Log() << "Processed " << max_internal_node_id << " nodes"; } void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environment) @@ -803,11 +849,10 @@ void ExtractionContainers::PrepareEdges(ScriptingEnvironment &scripting_environm all_edges_list[j].result.target = SPECIAL_NODEID; } } -} -void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) -{ - // std::vector normal_edges; + all_nodes_list.clear(); // free all_nodes_list before allocation of normal_edges + all_nodes_list.shrink_to_fit(); + normal_edges.reserve(all_edges_list.size()); { util::UnbufferedLog log; @@ -832,78 +877,12 @@ void ExtractionContainers::WriteEdges(storage::tar::FileWriter &writer) throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF); } - storage::serialization::write(writer, "/extractor/edges", normal_edges); - TIMER_STOP(write_edges); log << "ok, after " << TIMER_SEC(write_edges) << "s"; log << " -- Processed " << normal_edges.size() << " edges"; } } -void ExtractionContainers::WriteNodes(storage::tar::FileWriter &writer) -{ - { - util::UnbufferedLog log; - log << "Confirming/Writing used nodes ... "; - TIMER_START(write_nodes); - // identify all used nodes by a merging step of two sorted lists - auto node_iterator = all_nodes_list.begin(); - auto node_id_iterator = used_node_id_list.begin(); - const auto all_nodes_list_end = all_nodes_list.end(); - - const std::function encode_function = [&]() -> QueryNode { - BOOST_ASSERT(node_id_iterator != used_node_id_list.end()); - BOOST_ASSERT(node_iterator != all_nodes_list_end); - BOOST_ASSERT(*node_id_iterator >= node_iterator->node_id); - while (*node_id_iterator > node_iterator->node_id && - node_iterator != all_nodes_list_end) - { - ++node_iterator; - } - if (node_iterator == all_nodes_list_end || *node_id_iterator < node_iterator->node_id) - { - throw util::exception( - "Invalid OSM data: Referenced non-existing node with ID " + - std::to_string(static_cast(*node_id_iterator))); - } - BOOST_ASSERT(*node_id_iterator == node_iterator->node_id); - - ++node_id_iterator; - auto result = *node_iterator++; - - internal_nodes.push_back(result); - return result; - }; - - writer.WriteElementCount64("/extractor/nodes", used_node_id_list.size()); - writer.WriteStreaming( - "/extractor/nodes", - boost::make_function_input_iterator(encode_function, boost::infinite()), - used_node_id_list.size()); - - TIMER_STOP(write_nodes); - log << "ok, after " << TIMER_SEC(write_nodes) << "s"; - } - - { - util::UnbufferedLog log; - log << "Writing barrier nodes ... "; - TIMER_START(write_nodes); - for (const auto osm_id : barrier_nodes) - { - const auto node_id = mapExternalToInternalNodeID( - used_node_id_list.begin(), used_node_id_list.end(), osm_id); - if (node_id != SPECIAL_NODEID) - { - internal_barrier_nodes.emplace(node_id); - } - } - log << "ok, after " << TIMER_SEC(write_nodes) << "s"; - } - - util::Log() << "Processed " << max_internal_node_id << " nodes"; -} - ExtractionContainers::ReferencedWays ExtractionContainers::IdentifyManeuverOverrideWays() { ReferencedWays maneuver_override_ways; diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 5aea32004..e39801d6f 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -639,7 +639,6 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, } extraction_containers.PrepareData(scripting_environment, - config.GetPath(".osrm").string(), config.GetPath(".osrm.names").string()); auto profile_properties = scripting_environment.GetProfileProperties(); @@ -667,6 +666,16 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, osm_node_ids.push_back(current_node.node_id); } + if (config.dump_nbg_graph) + { + storage::tar::FileWriter writer(config.GetPath(".osrm").string(), + storage::tar::FileWriter::GenerateFingerprint); + storage::serialization::write( + writer, "/extractor/nodes", extraction_containers.internal_nodes); + storage::serialization::write( + writer, "/extractor/edges", extraction_containers.normal_edges); + } + return std::make_tuple(std::move(turn_lane_map), std::move(extraction_containers.turn_restrictions), std::move(extraction_containers.internal_maneuver_overrides), diff --git a/src/tools/extract.cpp b/src/tools/extract.cpp index 0cd1c7b4f..0fc615c49 100644 --- a/src/tools/extract.cpp +++ b/src/tools/extract.cpp @@ -74,7 +74,12 @@ return_code parseArguments(int argc, boost::program_options::bool_switch(&extractor_config.use_locations_cache) ->implicit_value(false) ->default_value(true), - "Use internal nodes locations cache for location-dependent data lookups"); + "Use internal nodes locations cache for location-dependent data lookups")( + "dump-nbg-graph", + boost::program_options::bool_switch(&extractor_config.dump_nbg_graph) + ->implicit_value(true) + ->default_value(false), + "Dump raw node-based graph to *.osrm file for debug purposes."); bool dummy; // hidden options, will be allowed on command line, but will not be