diff --git a/CMakeLists.txt b/CMakeLists.txt index 503171a59..6fbf409e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,7 +90,7 @@ set( add_library(COORDINATE OBJECT ${CoordinateGlob}) add_library(GITDESCRIPTION OBJECT util/git_sha.cpp) -add_library(OSRM ${OSRMSources} $ $ $ $ $ $ $ $ $) +add_library(OSRM ${OSRMSources} $ $ $ $ $ $ $ $ $ $) add_library(FINGERPRINT OBJECT util/fingerprint.cpp) add_dependencies(FINGERPRINT FingerPrintConfigure) @@ -328,12 +328,10 @@ if(WITH_TOOLS OR BUILD_TOOLS) if(UNIX AND NOT APPLE) target_link_libraries(osrm-unlock-all rt) endif() - add_executable(osrm-check-hsgr tools/check-hsgr.cpp $ $ $) + add_executable(osrm-check-hsgr tools/check-hsgr.cpp $ $ $ $) target_link_libraries(osrm-check-hsgr ${Boost_LIBRARIES}) add_executable(osrm-springclean tools/springclean.cpp $ $ $ $) target_link_libraries(osrm-springclean ${Boost_LIBRARIES}) - add_executable(osrm-graph-compare tools/graph_compare.cpp $ $ $ $ $ $ $) - target_link_libraries(osrm-graph-compare ${Boost_LIBRARIES} ${TBB_LIBRARIES}) install(TARGETS osrm-cli DESTINATION bin) install(TARGETS osrm-io-benchmark DESTINATION bin) diff --git a/contractor/contractor.hpp b/contractor/contractor.hpp index ccfabbc8f..e4a278c71 100644 --- a/contractor/contractor.hpp +++ b/contractor/contractor.hpp @@ -186,6 +186,7 @@ class Contractor } // clear input vector input_edge_list.clear(); + // FIXME not sure if we need this edges.shrink_to_fit(); tbb::parallel_sort(edges.begin(), edges.end()); @@ -953,7 +954,6 @@ class Contractor } std::shared_ptr contractor_graph; - std::vector contracted_edge_list; stxxl::vector external_edge_list; std::vector orig_node_id_to_new_id_map; XORFastHash fast_hash; diff --git a/contractor/contractor_options.cpp b/contractor/contractor_options.cpp new file mode 100644 index 000000000..175613f03 --- /dev/null +++ b/contractor/contractor_options.cpp @@ -0,0 +1,136 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "contractor_options.hpp" + +#include "../util/git_sha.hpp" +#include "../util/ini_file.hpp" +#include "../util/simple_logger.hpp" + +#include +#include + +#include + +return_code +ContractorOptions::ParseArguments(int argc, char *argv[], ContractorConfig &contractor_config) +{ + // declare a group of options that will be allowed only on command line + boost::program_options::options_description generic_options("Options"); + generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message")( + "config,c", boost::program_options::value(&contractor_config.config_file_path) + ->default_value("contractor.ini"), + "Path to a configuration file."); + + // declare a group of options that will be allowed both on command line and in config file + boost::program_options::options_description config_options("Configuration"); + config_options.add_options()( + "restrictions,r", + boost::program_options::value(&contractor_config.restrictions_path), + "Restrictions file in .osrm.restrictions format")( + "profile,p", boost::program_options::value(&contractor_config.profile_path) + ->default_value("profile.lua"), + "Path to LUA routing profile")( + "threads,t", boost::program_options::value(&contractor_config.requested_num_threads) + ->default_value(tbb::task_scheduler_init::default_num_threads()), + "Number of threads to use"); + + // hidden options, will be allowed both on command line and in config file, but will not be + // shown to the user + boost::program_options::options_description hidden_options("Hidden options"); + hidden_options.add_options()( + "input,i", boost::program_options::value(&contractor_config.osrm_input_path), + "Input file in .osm, .osm.bz2 or .osm.pbf format"); + + // positional option + boost::program_options::positional_options_description positional_options; + positional_options.add("input", 1); + + // combine above options for parsing + boost::program_options::options_description cmdline_options; + cmdline_options.add(generic_options).add(config_options).add(hidden_options); + + boost::program_options::options_description config_file_options; + config_file_options.add(config_options).add(hidden_options); + + boost::program_options::options_description visible_options( + "Usage: " + boost::filesystem::basename(argv[0]) + " [options]"); + visible_options.add(generic_options).add(config_options); + + // parse command line options + boost::program_options::variables_map option_variables; + boost::program_options::store(boost::program_options::command_line_parser(argc, argv) + .options(cmdline_options) + .positional(positional_options) + .run(), + option_variables); + + const auto &temp_config_path = option_variables["config"].as(); + if (boost::filesystem::is_regular_file(temp_config_path)) + { + boost::program_options::store(boost::program_options::parse_config_file( + temp_config_path.string().c_str(), cmdline_options, true), + option_variables); + } + + if (option_variables.count("version")) + { + SimpleLogger().Write() << g_GIT_DESCRIPTION; + return return_code::exit; + } + + if (option_variables.count("help")) + { + SimpleLogger().Write() << "\n" << visible_options; + return return_code::exit; + } + + boost::program_options::notify(option_variables); + + if (!option_variables.count("restrictions")) + { + contractor_config.restrictions_path = contractor_config.osrm_input_path.string() + ".restrictions"; + } + + if (!option_variables.count("input")) + { + SimpleLogger().Write() << "\n" << visible_options; + return return_code::fail; + } + + return return_code::ok; +} + +void ContractorOptions::GenerateOutputFilesNames(ContractorConfig &contractor_config) +{ + contractor_config.node_output_path = contractor_config.osrm_input_path.string() + ".nodes"; + contractor_config.edge_output_path = contractor_config.osrm_input_path.string() + ".edges"; + contractor_config.geometry_output_path = contractor_config.osrm_input_path.string() + ".geometry"; + contractor_config.graph_output_path = contractor_config.osrm_input_path.string() + ".hsgr"; + contractor_config.rtree_nodes_output_path = contractor_config.osrm_input_path.string() + ".ramIndex"; + contractor_config.rtree_leafs_output_path = contractor_config.osrm_input_path.string() + ".fileIndex"; +} diff --git a/contractor/contractor_options.hpp b/contractor/contractor_options.hpp new file mode 100644 index 000000000..87541af9d --- /dev/null +++ b/contractor/contractor_options.hpp @@ -0,0 +1,68 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef CONTRACTOR_OPTIONS_HPP +#define CONTRACTOR_OPTIONS_HPP + +#include + +#include + +enum class return_code : unsigned +{ + ok, + fail, + exit +}; + +struct ContractorConfig +{ + ContractorConfig() noexcept : requested_num_threads(0) {} + + boost::filesystem::path config_file_path; + boost::filesystem::path osrm_input_path; + boost::filesystem::path restrictions_path; + boost::filesystem::path profile_path; + + std::string node_output_path; + std::string edge_output_path; + std::string geometry_output_path; + std::string graph_output_path; + std::string rtree_nodes_output_path; + std::string rtree_leafs_output_path; + + unsigned requested_num_threads; +}; + +struct ContractorOptions +{ + static return_code ParseArguments(int argc, char *argv[], ContractorConfig &extractor_config); + + static void GenerateOutputFilesNames(ContractorConfig &extractor_config); +}; + +#endif // EXTRACTOR_OPTIONS_HPP diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index 268bb9f8d..c760f0f43 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -40,21 +40,21 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -EdgeBasedGraphFactory::EdgeBasedGraphFactory( - const std::shared_ptr &node_based_graph, - std::unique_ptr restriction_map, - std::vector &barrier_node_list, - std::vector &traffic_light_node_list, - std::vector &node_info_list, - SpeedProfileProperties &speed_profile) +EdgeBasedGraphFactory::EdgeBasedGraphFactory(std::shared_ptr node_based_graph, + std::shared_ptr restriction_map, + std::unique_ptr> barrier_node_list, + std::unique_ptr> traffic_light_node_list, + const std::vector &node_info_list, + const SpeedProfileProperties &speed_profile) : speed_profile(speed_profile), m_number_of_edge_based_nodes(std::numeric_limits::max()), - m_node_info_list(node_info_list), m_node_based_graph(node_based_graph), + m_node_info_list(node_info_list), + m_node_based_graph(std::move(node_based_graph)), m_restriction_map(std::move(restriction_map)), max_id(0), removed_node_count(0) { // insert into unordered sets for fast lookup - m_barrier_nodes.insert(barrier_node_list.begin(), barrier_node_list.end()); - m_traffic_lights.insert(traffic_light_node_list.begin(), traffic_light_node_list.end()); + m_barrier_nodes.insert(barrier_node_list->begin(), barrier_node_list->end()); + m_traffic_lights.insert(traffic_light_node_list->begin(), traffic_light_node_list->end()); } void EdgeBasedGraphFactory::GetEdgeBasedEdges(DeallocatingVector &output_edge_list) @@ -290,12 +290,33 @@ void EdgeBasedGraphFactory::CompressGeometry() continue; } + /* + * reverse_e2 forward_e2 + * u <---------- v -----------> w + * ----------> <----------- + * forward_e1 reverse_e1 + * + * Will be compressed to: + * + * reverse_e1 + * u <---------- w + * ----------> + * forward_e1 + * + * If the edges are compatible. + * + */ + const bool reverse_edge_order = !(m_node_based_graph->GetEdgeData(m_node_based_graph->BeginEdges(node_v)).forward); const EdgeID forward_e2 = m_node_based_graph->BeginEdges(node_v) + reverse_edge_order; BOOST_ASSERT(SPECIAL_EDGEID != forward_e2); + BOOST_ASSERT(forward_e2 >= m_node_based_graph->BeginEdges(node_v) && + forward_e2 < m_node_based_graph->EndEdges(node_v)); const EdgeID reverse_e2 = m_node_based_graph->BeginEdges(node_v) + 1 - reverse_edge_order; BOOST_ASSERT(SPECIAL_EDGEID != reverse_e2); + BOOST_ASSERT(reverse_e2 >= m_node_based_graph->BeginEdges(node_v) && + reverse_e2 < m_node_based_graph->EndEdges(node_v)); const EdgeData &fwd_edge_data2 = m_node_based_graph->GetEdgeData(forward_e2); const EdgeData &rev_edge_data2 = m_node_based_graph->GetEdgeData(reverse_e2); @@ -325,6 +346,11 @@ void EdgeBasedGraphFactory::CompressGeometry() if ( // TODO: rename to IsCompatibleTo fwd_edge_data1.IsEqualTo(fwd_edge_data2) && rev_edge_data1.IsEqualTo(rev_edge_data2)) { + BOOST_ASSERT(m_node_based_graph->GetEdgeData(forward_e1).nameID == + m_node_based_graph->GetEdgeData(reverse_e1).nameID); + BOOST_ASSERT(m_node_based_graph->GetEdgeData(forward_e2).nameID == + m_node_based_graph->GetEdgeData(reverse_e2).nameID); + // Get distances before graph is modified const int forward_weight1 = m_node_based_graph->GetEdgeData(forward_e1).distance; const int forward_weight2 = m_node_based_graph->GetEdgeData(forward_e2).distance; @@ -336,7 +362,7 @@ void EdgeBasedGraphFactory::CompressGeometry() const int reverse_weight2 = m_node_based_graph->GetEdgeData(reverse_e2).distance; BOOST_ASSERT(0 != reverse_weight1); - BOOST_ASSERT(0 != forward_weight2); + BOOST_ASSERT(0 != reverse_weight2); const bool has_node_penalty = m_traffic_lights.find(node_v) != m_traffic_lights.end(); @@ -362,11 +388,11 @@ void EdgeBasedGraphFactory::CompressGeometry() // update any involved turn restrictions m_restriction_map->FixupStartingTurnRestriction(node_u, node_v, node_w); m_restriction_map->FixupArrivingTurnRestriction(node_u, node_v, node_w, - m_node_based_graph); + *m_node_based_graph); m_restriction_map->FixupStartingTurnRestriction(node_w, node_v, node_u); m_restriction_map->FixupArrivingTurnRestriction(node_w, node_v, node_u, - m_node_based_graph); + *m_node_based_graph); // store compressed geometry in container m_geometry_compressor.CompressEdge( @@ -378,8 +404,6 @@ void EdgeBasedGraphFactory::CompressGeometry() reverse_weight2 + (has_node_penalty ? speed_profile.traffic_signal_penalty : 0)); ++removed_node_count; - BOOST_ASSERT(m_node_based_graph->GetEdgeData(forward_e1).nameID == - m_node_based_graph->GetEdgeData(reverse_e1).nameID); } } SimpleLogger().Write() << "removed " << removed_node_count << " nodes"; @@ -415,6 +439,7 @@ void EdgeBasedGraphFactory::RenumberEdges() for (const auto current_edge : m_node_based_graph->GetAdjacentEdgeRange(current_node)) { EdgeData &edge_data = m_node_based_graph->GetEdgeData(current_edge); + // FIXME when does that happen? why can we skip here? if (!edge_data.forward) { continue; diff --git a/contractor/edge_based_graph_factory.hpp b/contractor/edge_based_graph_factory.hpp index 95b65babc..a5274396f 100644 --- a/contractor/edge_based_graph_factory.hpp +++ b/contractor/edge_based_graph_factory.hpp @@ -59,12 +59,12 @@ class EdgeBasedGraphFactory struct SpeedProfileProperties; - explicit EdgeBasedGraphFactory(const std::shared_ptr &node_based_graph, - std::unique_ptr restricion_map, - std::vector &barrier_node_list, - std::vector &traffic_light_node_list, - std::vector &node_info_list, - SpeedProfileProperties &speed_profile); + explicit EdgeBasedGraphFactory(std::shared_ptr node_based_graph, + std::shared_ptr restricion_map, + std::unique_ptr> barrier_node_list, + std::unique_ptr> traffic_light_node_list, + const std::vector &node_info_list, + const SpeedProfileProperties &speed_profile); void Run(const std::string &original_edge_data_filename, const std::string &geometry_filename, @@ -97,15 +97,16 @@ class EdgeBasedGraphFactory unsigned m_number_of_edge_based_nodes; - std::vector m_node_info_list; std::vector m_edge_based_node_list; DeallocatingVector m_edge_based_edge_list; + const std::vector& m_node_info_list; std::shared_ptr m_node_based_graph; + std::shared_ptr m_restriction_map; + std::unordered_set m_barrier_nodes; std::unordered_set m_traffic_lights; - std::unique_ptr m_restriction_map; GeometryCompressor m_geometry_compressor; diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index ccff7fd09..567ef272a 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -48,7 +48,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#include #include #include @@ -57,166 +56,98 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -Prepare::Prepare() : requested_num_threads(1) {} - Prepare::~Prepare() {} -int Prepare::Process(int argc, char *argv[]) +int Prepare::Run() { - LogPolicy::GetInstance().Unmute(); - TIMER_START(preparing); - TIMER_START(expansion); - - if (!ParseArguments(argc, argv)) - { - return 0; - } - if (!boost::filesystem::is_regular_file(input_path)) - { - SimpleLogger().Write(logWARNING) << "Input file " << input_path.string() << " not found!"; - return 1; - } - - if (!boost::filesystem::is_regular_file(profile_path)) - { - SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string() << " not found!"; - return 1; - } - - if (1 > requested_num_threads) - { - SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger"; - return 1; - } - - const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); - - SimpleLogger().Write() << "Input file: " << input_path.filename().string(); - SimpleLogger().Write() << "Restrictions file: " << restrictions_path.filename().string(); - SimpleLogger().Write() << "Profile: " << profile_path.filename().string(); - SimpleLogger().Write() << "Threads: " << requested_num_threads; - if (recommended_num_threads != requested_num_threads) - { - SimpleLogger().Write(logWARNING) << "The recommended number of threads is " - << recommended_num_threads - << "! This setting may have performance side-effects."; - } - - tbb::task_scheduler_init init(requested_num_threads); - - LogPolicy::GetInstance().Unmute(); - - FingerPrint fingerprint_orig; - CheckRestrictionsFile(fingerprint_orig); - - boost::filesystem::ifstream input_stream(input_path, std::ios::in | std::ios::binary); - - node_filename = input_path.string() + ".nodes"; - edge_out = input_path.string() + ".edges"; - geometry_filename = input_path.string() + ".geometry"; - graph_out = input_path.string() + ".hsgr"; - rtree_nodes_path = input_path.string() + ".ramIndex"; - rtree_leafs_path = input_path.string() + ".fileIndex"; - - /*** Setup Scripting Environment ***/ - // Create a new lua state - lua_State *lua_state = luaL_newstate(); - - // Connect LuaBind to this lua state - luabind::open(lua_state); - - EdgeBasedGraphFactory::SpeedProfileProperties speed_profile; - - if (!SetupScriptingEnvironment(lua_state, speed_profile)) - { - return 1; - } - #ifdef WIN32 #pragma message("Memory consumption on Windows can be higher due to different bit packing") #else static_assert(sizeof(ImportEdge) == 20, "changing ImportEdge type has influence on memory consumption!"); + static_assert(sizeof(EdgeBasedEdge) == 16, + "changing EdgeBasedEdge type has influence on memory consumption!"); #endif - NodeID number_of_node_based_nodes = readBinaryOSRMGraphFromStream( - input_stream, edge_list, barrier_node_list, traffic_light_list, - &internal_to_external_node_map, restriction_list); - input_stream.close(); - if (edge_list.empty()) - { - SimpleLogger().Write(logWARNING) << "The input data is empty, exiting."; - return 1; - } + TIMER_START(preparing); - SimpleLogger().Write() << restriction_list.size() << " restrictions, " - << barrier_node_list.size() << " bollard nodes, " - << traffic_light_list.size() << " traffic lights"; + // Create a new lua state - std::vector node_based_edge_list; - unsigned number_of_edge_based_nodes = 0; + SimpleLogger().Write() << "Generating edge-expanded graph representation"; + + TIMER_START(expansion); + + auto node_based_edge_list = osrm::make_unique>();; DeallocatingVector edge_based_edge_list; + auto internal_to_external_node_map = osrm::make_unique>(); + auto graph_size = + BuildEdgeExpandedGraph(*internal_to_external_node_map, + *node_based_edge_list, edge_based_edge_list); - // init node_based_edge_list, edge_based_edge_list by edgeList - number_of_edge_based_nodes = - BuildEdgeExpandedGraph(lua_state, number_of_node_based_nodes, node_based_edge_list, - edge_based_edge_list, speed_profile); - lua_close(lua_state); + auto number_of_node_based_nodes = graph_size.first; + auto number_of_edge_based_nodes = graph_size.second; TIMER_STOP(expansion); - BuildRTree(node_based_edge_list); + SimpleLogger().Write() << "building r-tree ..."; + TIMER_START(rtree); - RangebasedCRC32 crc32; - if (crc32.using_hardware()) - { - SimpleLogger().Write() << "using hardware based CRC32 computation"; - } - else - { - SimpleLogger().Write() << "using software based CRC32 computation"; - } + BuildRTree(*node_based_edge_list, *internal_to_external_node_map); - const unsigned crc32_value = crc32(node_based_edge_list); - node_based_edge_list.clear(); - node_based_edge_list.shrink_to_fit(); - SimpleLogger().Write() << "CRC32: " << crc32_value; + TIMER_STOP(rtree); - WriteNodeMapping(); + SimpleLogger().Write() << "writing node map ..."; + WriteNodeMapping(std::move(internal_to_external_node_map)); - /*** - * Contracting the edge-expanded graph - */ - - SimpleLogger().Write() << "initializing contractor"; - auto contractor = - osrm::make_unique(number_of_edge_based_nodes, edge_based_edge_list); + // Contracting the edge-expanded graph TIMER_START(contraction); - contractor->Run(); + auto contracted_edge_list = osrm::make_unique>(); + ContractGraph(number_of_edge_based_nodes, edge_based_edge_list, *contracted_edge_list); TIMER_STOP(contraction); SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec"; - DeallocatingVector contracted_edge_list; - contractor->GetEdges(contracted_edge_list); - contractor.reset(); + std::size_t number_of_used_edges = WriteContractedGraph(number_of_edge_based_nodes, + std::move(node_based_edge_list), + std::move(contracted_edge_list)); - /*** - * Sorting contracted edges in a way that the static query graph can read some in in-place. - */ + TIMER_STOP(preparing); - tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end()); - const unsigned contracted_edge_count = contracted_edge_list.size(); + SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds"; + SimpleLogger().Write() << "Expansion : " << (number_of_node_based_nodes / TIMER_SEC(expansion)) + << " nodes/sec and " + << (number_of_edge_based_nodes / TIMER_SEC(expansion)) << " edges/sec"; + + SimpleLogger().Write() << "Contraction: " + << (number_of_edge_based_nodes / TIMER_SEC(contraction)) + << " nodes/sec and " << number_of_used_edges / TIMER_SEC(contraction) + << " edges/sec"; + + SimpleLogger().Write() << "finished preprocessing"; + + return 0; +} + +std::size_t Prepare::WriteContractedGraph(unsigned number_of_edge_based_nodes, + std::unique_ptr> node_based_edge_list, + std::unique_ptr> contracted_edge_list) +{ + const unsigned crc32_value = CalculateEdgeChecksum(std::move(node_based_edge_list)); + + // Sorting contracted edges in a way that the static query graph can read some in in-place. + tbb::parallel_sort(contracted_edge_list->begin(), contracted_edge_list->end()); + const unsigned contracted_edge_count = contracted_edge_list->size(); SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count << " edges"; - boost::filesystem::ofstream hsgr_output_stream(graph_out, std::ios::binary); + FingerPrint fingerprint_orig; + boost::filesystem::ofstream hsgr_output_stream(config.graph_output_path, std::ios::binary); hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint)); const unsigned max_used_node_id = 1 + [&contracted_edge_list] { unsigned tmp_max = 0; - for (const QueryEdge &edge : contracted_edge_list) + for (const QueryEdge &edge : *contracted_edge_list) { BOOST_ASSERT(SPECIAL_NODEID != edge.source); BOOST_ASSERT(SPECIAL_NODEID != edge.target); @@ -241,7 +172,7 @@ int Prepare::Process(int argc, char *argv[]) for (const auto node : osrm::irange(0u, max_used_node_id)) { last_edge = edge; - while ((edge < contracted_edge_count) && (contracted_edge_list[edge].source == node)) + while ((edge < contracted_edge_count) && ((*contracted_edge_list)[edge].source == node)) { ++edge; } @@ -270,19 +201,19 @@ int Prepare::Process(int argc, char *argv[]) hsgr_output_stream.write((char *)&node_array[0], sizeof(StaticGraph::NodeArrayEntry) * node_array_size); } - // serialize all edges + // serialize all edges SimpleLogger().Write() << "Building edge array"; edge = 0; int number_of_used_edges = 0; StaticGraph::EdgeArrayEntry current_edge; - for (const auto edge : osrm::irange(0, contracted_edge_list.size())) + for (const auto edge : osrm::irange(0, contracted_edge_list->size())) { // no eigen loops - BOOST_ASSERT(contracted_edge_list[edge].source != contracted_edge_list[edge].target); - current_edge.target = contracted_edge_list[edge].target; - current_edge.data = contracted_edge_list[edge].data; + BOOST_ASSERT((*contracted_edge_list)[edge].source != (*contracted_edge_list)[edge].target); + current_edge.target = (*contracted_edge_list)[edge].target; + current_edge.data = (*contracted_edge_list)[edge].data; // every target needs to be valid BOOST_ASSERT(current_edge.target < max_used_node_id); @@ -290,12 +221,12 @@ int Prepare::Process(int argc, char *argv[]) if (current_edge.data.distance <= 0) { SimpleLogger().Write(logWARNING) << "Edge: " << edge - << ",source: " << contracted_edge_list[edge].source - << ", target: " << contracted_edge_list[edge].target + << ",source: " << (*contracted_edge_list)[edge].source + << ", target: " << (*contracted_edge_list)[edge].target << ", dist: " << current_edge.data.distance; SimpleLogger().Write(logWARNING) << "Failed at adjacency list of node " - << contracted_edge_list[edge].source << "/" + << (*contracted_edge_list)[edge].source << "/" << node_array.size() - 1; return 1; } @@ -305,170 +236,54 @@ int Prepare::Process(int argc, char *argv[]) ++number_of_used_edges; } - hsgr_output_stream.close(); - TIMER_STOP(preparing); - - SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds"; - SimpleLogger().Write() << "Expansion : " << (number_of_node_based_nodes / TIMER_SEC(expansion)) - << " nodes/sec and " - << (number_of_edge_based_nodes / TIMER_SEC(expansion)) << " edges/sec"; - - SimpleLogger().Write() << "Contraction: " - << (number_of_edge_based_nodes / TIMER_SEC(contraction)) - << " nodes/sec and " << number_of_used_edges / TIMER_SEC(contraction) - << " edges/sec"; - - node_array.clear(); - SimpleLogger().Write() << "finished preprocessing"; - - return 0; + return number_of_used_edges; } -/** - \brief Parses command line arguments - \param argc count of arguments - \param argv array of arguments - \param result [out] value for exit return value - \return true if everything is ok, false if need to terminate execution -*/ -bool Prepare::ParseArguments(int argc, char *argv[]) +unsigned Prepare::CalculateEdgeChecksum(std::unique_ptr> node_based_edge_list) { - // declare a group of options that will be allowed only on command line - boost::program_options::options_description generic_options("Options"); - generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message")( - "config,c", boost::program_options::value(&config_file_path) - ->default_value("contractor.ini"), - "Path to a configuration file."); - - // declare a group of options that will be allowed both on command line and in config file - boost::program_options::options_description config_options("Configuration"); - config_options.add_options()( - "restrictions,r", - boost::program_options::value(&restrictions_path), - "Restrictions file in .osrm.restrictions format")( - "profile,p", boost::program_options::value(&profile_path) - ->default_value("profile.lua"), - "Path to LUA routing profile")( - "threads,t", boost::program_options::value(&requested_num_threads) - ->default_value(tbb::task_scheduler_init::default_num_threads()), - "Number of threads to use"); - - // hidden options, will be allowed both on command line and in config file, but will not be - // shown to the user - boost::program_options::options_description hidden_options("Hidden options"); - hidden_options.add_options()( - "input,i", boost::program_options::value(&input_path), - "Input file in .osm, .osm.bz2 or .osm.pbf format"); - - // positional option - boost::program_options::positional_options_description positional_options; - positional_options.add("input", 1); - - // combine above options for parsing - boost::program_options::options_description cmdline_options; - cmdline_options.add(generic_options).add(config_options).add(hidden_options); - - boost::program_options::options_description config_file_options; - config_file_options.add(config_options).add(hidden_options); - - boost::program_options::options_description visible_options( - "Usage: " + boost::filesystem::basename(argv[0]) + " [options]"); - visible_options.add(generic_options).add(config_options); - - // parse command line options - boost::program_options::variables_map option_variables; - boost::program_options::store(boost::program_options::command_line_parser(argc, argv) - .options(cmdline_options) - .positional(positional_options) - .run(), - option_variables); - - const auto &temp_config_path = option_variables["config"].as(); - if (boost::filesystem::is_regular_file(temp_config_path)) + RangebasedCRC32 crc32; + if (crc32.using_hardware()) { - boost::program_options::store(boost::program_options::parse_config_file( - temp_config_path.string().c_str(), cmdline_options, true), - option_variables); + SimpleLogger().Write() << "using hardware based CRC32 computation"; + } + else + { + SimpleLogger().Write() << "using software based CRC32 computation"; } - if (option_variables.count("version")) - { - SimpleLogger().Write() << g_GIT_DESCRIPTION; - return false; - } + const unsigned crc32_value = crc32(*node_based_edge_list); + SimpleLogger().Write() << "CRC32: " << crc32_value; - if (option_variables.count("help")) - { - SimpleLogger().Write() << "\n" << visible_options; - return false; - } - - boost::program_options::notify(option_variables); - - if (!option_variables.count("restrictions")) - { - restrictions_path = std::string(input_path.string() + ".restrictions"); - } - - if (!option_variables.count("input")) - { - SimpleLogger().Write() << "\n" << visible_options; - return false; - } - - return true; -} - -/** - \brief Loads and checks file UUIDs -*/ -void Prepare::CheckRestrictionsFile(FingerPrint &fingerprint_orig) -{ - boost::filesystem::ifstream restriction_stream(restrictions_path, std::ios::binary); - FingerPrint fingerprint_loaded; - unsigned number_of_usable_restrictions = 0; - restriction_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint)); - if (!fingerprint_loaded.TestPrepare(fingerprint_orig)) - { - SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n" - "Reprocess to get rid of this warning."; - } - - restriction_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned)); - restriction_list.resize(number_of_usable_restrictions); - if (number_of_usable_restrictions > 0) - { - restriction_stream.read((char *)&(restriction_list[0]), - number_of_usable_restrictions * sizeof(TurnRestriction)); - } - restriction_stream.close(); + return crc32_value; } /** \brief Setups scripting environment (lua-scripting) Also initializes speed profile. */ -bool Prepare::SetupScriptingEnvironment( +void Prepare::SetupScriptingEnvironment( lua_State *lua_state, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile) { // open utility libraries string library; luaL_openlibs(lua_state); // adjust lua load path - luaAddScriptFolderToLoadPath(lua_state, profile_path.string().c_str()); + luaAddScriptFolderToLoadPath(lua_state, config.profile_path.string().c_str()); // Now call our function in a lua script - if (0 != luaL_dofile(lua_state, profile_path.string().c_str())) + if (0 != luaL_dofile(lua_state, config.profile_path.string().c_str())) { - std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; - return false; + std::stringstream msg; + msg << lua_tostring(lua_state, -1) << " occured in scripting block"; + throw osrm::exception(msg.str()); } if (0 != luaL_dostring(lua_state, "return traffic_signal_penalty\n")) { - std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; - return false; + std::stringstream msg; + msg << lua_tostring(lua_state, -1) << " occured in scripting block"; + throw osrm::exception(msg.str()); } speed_profile.traffic_signal_penalty = 10 * lua_tointeger(lua_state, -1); SimpleLogger().Write(logDEBUG) @@ -476,92 +291,146 @@ bool Prepare::SetupScriptingEnvironment( if (0 != luaL_dostring(lua_state, "return u_turn_penalty\n")) { - std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; - return false; + std::stringstream msg; + msg << lua_tostring(lua_state, -1) << " occured in scripting block"; + throw osrm::exception(msg.str()); } speed_profile.u_turn_penalty = 10 * lua_tointeger(lua_state, -1); speed_profile.has_turn_penalty_function = lua_function_exists(lua_state, "turn_function"); - - return true; } +/** + \brief Build load restrictions from .restriction file + */ +void Prepare::LoadRestrictionMap(const std::unordered_map &external_to_internal_node_map, + RestrictionMap &restriction_map) +{ + boost::filesystem::ifstream input_stream(config.restrictions_path, std::ios::in | std::ios::binary); + + std::vector restriction_list; + loadRestrictionsFromFile(input_stream, external_to_internal_node_map, restriction_list); + + SimpleLogger().Write() << " - " << restriction_list.size() << " restrictions."; + + restriction_map = RestrictionMap(restriction_list); +} + +/** + \brief Build load node based graph from .osrm file and restrictions from .restrictions file + */ +std::shared_ptr +Prepare::LoadNodeBasedGraph(std::vector &barrier_node_list, + std::vector &traffic_light_list, + RestrictionMap &restriction_map, + std::vector& internal_to_external_node_map) +{ + std::vector edge_list; + std::unordered_map external_to_internal_node_map; + + boost::filesystem::ifstream input_stream(config.osrm_input_path, std::ios::in | std::ios::binary); + + NodeID number_of_node_based_nodes = loadNodesFromFile(input_stream, + barrier_node_list, traffic_light_list, + internal_to_external_node_map, + external_to_internal_node_map); + + SimpleLogger().Write() << " - " << barrier_node_list.size() << " bollard nodes, " + << traffic_light_list.size() << " traffic lights"; + + loadEdgesFromFile(input_stream, external_to_internal_node_map, edge_list); + + LoadRestrictionMap(external_to_internal_node_map, restriction_map); + + if (edge_list.empty()) + { + SimpleLogger().Write(logWARNING) << "The input data is empty, exiting."; + return std::shared_ptr(); + } + + return NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list); +} + + /** \brief Building an edge-expanded graph from node-based input and turn restrictions */ -std::size_t -Prepare::BuildEdgeExpandedGraph(lua_State *lua_state, - NodeID number_of_node_based_nodes, +std::pair +Prepare::BuildEdgeExpandedGraph(std::vector &internal_to_external_node_map, std::vector &node_based_edge_list, - DeallocatingVector &edge_based_edge_list, - EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile) + DeallocatingVector &edge_based_edge_list) { - SimpleLogger().Write() << "Generating edge-expanded graph representation"; - std::shared_ptr node_based_graph = - NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list); - std::unique_ptr restriction_map = - osrm::make_unique(restriction_list); - std::shared_ptr edge_based_graph_factory = - std::make_shared(node_based_graph, std::move(restriction_map), - barrier_node_list, traffic_light_list, - internal_to_external_node_map, speed_profile); - edge_list.clear(); - edge_list.shrink_to_fit(); + lua_State *lua_state = luaL_newstate(); + luabind::open(lua_state); - edge_based_graph_factory->Run(edge_out, geometry_filename, lua_state); + EdgeBasedGraphFactory::SpeedProfileProperties speed_profile; - restriction_list.clear(); - restriction_list.shrink_to_fit(); - barrier_node_list.clear(); - barrier_node_list.shrink_to_fit(); - traffic_light_list.clear(); - traffic_light_list.shrink_to_fit(); + SetupScriptingEnvironment(lua_state, speed_profile); + + auto barrier_node_list = osrm::make_unique>(); + auto traffic_light_list = osrm::make_unique>(); + auto restriction_map = std::make_shared(); + + auto node_based_graph = LoadNodeBasedGraph(*barrier_node_list, *traffic_light_list, *restriction_map, internal_to_external_node_map); + + const std::size_t number_of_node_based_nodes = node_based_graph->GetNumberOfNodes(); + + EdgeBasedGraphFactory edge_based_graph_factory(node_based_graph, + restriction_map, + std::move(barrier_node_list), + std::move(traffic_light_list), + internal_to_external_node_map, + speed_profile); + + edge_based_graph_factory.Run(config.edge_output_path, config.geometry_output_path, lua_state); + lua_close(lua_state); const std::size_t number_of_edge_based_nodes = - edge_based_graph_factory->GetNumberOfEdgeBasedNodes(); + edge_based_graph_factory.GetNumberOfEdgeBasedNodes(); BOOST_ASSERT(number_of_edge_based_nodes != std::numeric_limits::max()); -#ifndef WIN32 - static_assert(sizeof(EdgeBasedEdge) == 16, - "changing ImportEdge type has influence on memory consumption!"); -#endif - edge_based_graph_factory->GetEdgeBasedEdges(edge_based_edge_list); - edge_based_graph_factory->GetEdgeBasedNodes(node_based_edge_list); + edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list); + edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list); - edge_based_graph_factory.reset(); - node_based_graph.reset(); + return std::make_pair(number_of_node_based_nodes, number_of_edge_based_nodes); +} - return number_of_edge_based_nodes; +/** + \brief Build contracted graph. + */ +void Prepare::ContractGraph(const std::size_t number_of_edge_based_nodes, + DeallocatingVector& edge_based_edge_list, + DeallocatingVector& contracted_edge_list) +{ + Contractor contractor(number_of_edge_based_nodes, edge_based_edge_list); + contractor.Run(); + contractor.GetEdges(contracted_edge_list); } /** \brief Writing info on original (node-based) nodes */ -void Prepare::WriteNodeMapping() +void Prepare::WriteNodeMapping(std::unique_ptr> internal_to_external_node_map) { - SimpleLogger().Write() << "writing node map ..."; - boost::filesystem::ofstream node_stream(node_filename, std::ios::binary); - const unsigned size_of_mapping = internal_to_external_node_map.size(); + boost::filesystem::ofstream node_stream(config.node_output_path, std::ios::binary); + const unsigned size_of_mapping = internal_to_external_node_map->size(); node_stream.write((char *)&size_of_mapping, sizeof(unsigned)); if (size_of_mapping > 0) { - node_stream.write((char *)&(internal_to_external_node_map[0]), + node_stream.write((char *) internal_to_external_node_map->data(), size_of_mapping * sizeof(QueryNode)); } node_stream.close(); - internal_to_external_node_map.clear(); - internal_to_external_node_map.shrink_to_fit(); } /** \brief Building rtree-based nearest-neighbor data structure - Saves info to files: '.ramIndex' and '.fileIndex'. + Saves tree into '.ramIndex' and leaves into '.fileIndex'. */ -void Prepare::BuildRTree(std::vector &node_based_edge_list) +void Prepare::BuildRTree(const std::vector &node_based_edge_list, const std::vector& internal_to_external_node_map) { - SimpleLogger().Write() << "building r-tree ..."; - StaticRTree(node_based_edge_list, rtree_nodes_path.c_str(), - rtree_leafs_path.c_str(), internal_to_external_node_map); + StaticRTree(node_based_edge_list, config.rtree_nodes_output_path.c_str(), + config.rtree_leafs_output_path.c_str(), internal_to_external_node_map); } diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index 933213a4c..2e4b2cd1f 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -28,6 +28,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef PROCESSING_CHAIN_HPP #define PROCESSING_CHAIN_HPP +#include "contractor_options.hpp" #include "edge_based_graph_factory.hpp" #include "../data_structures/query_edge.hpp" #include "../data_structures/static_graph.hpp" @@ -50,46 +51,38 @@ class Prepare using InputEdge = DynamicGraph::InputEdge; using StaticEdge = StaticGraph::InputEdge; - explicit Prepare(); + explicit Prepare(const ContractorConfig& contractor_config) + : config(contractor_config) {} Prepare(const Prepare &) = delete; ~Prepare(); - int Process(int argc, char *argv[]); + int Run(); protected: - bool ParseArguments(int argc, char *argv[]); - void CheckRestrictionsFile(FingerPrint &fingerprint_orig); - bool SetupScriptingEnvironment(lua_State *myLuaState, + void SetupScriptingEnvironment(lua_State *myLuaState, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile); - std::size_t BuildEdgeExpandedGraph(lua_State *myLuaState, - NodeID nodeBasedNodeNumber, - std::vector &nodeBasedEdgeList, - DeallocatingVector &edgeBasedEdgeList, - EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile); - void WriteNodeMapping(); - void BuildRTree(std::vector &node_based_edge_list); - + void LoadRestrictionMap(const std::unordered_map &external_to_internal_node_map, + RestrictionMap &restriction_map); + unsigned CalculateEdgeChecksum(std::unique_ptr> node_based_edge_list); + void ContractGraph(const std::size_t number_of_edge_based_nodes, + DeallocatingVector& edge_based_edge_list, + DeallocatingVector& contracted_edge_list); + std::size_t WriteContractedGraph(unsigned number_of_edge_based_nodes, + std::unique_ptr> node_based_edge_list, + std::unique_ptr> contracted_edge_list); + std::shared_ptr LoadNodeBasedGraph(std::vector &barrier_node_list, + std::vector &traffic_light_list, + RestrictionMap &restriction_map, + std::vector& internal_to_external_node_map); + std::pair + BuildEdgeExpandedGraph(std::vector &internal_to_external_node_map, + std::vector &node_based_edge_list, + DeallocatingVector &edge_based_edge_list); + void WriteNodeMapping(std::unique_ptr> internal_to_external_node_map); + void BuildRTree(const std::vector &node_based_edge_list, + const std::vector &internal_to_external_node_map); private: - std::vector internal_to_external_node_map; - std::vector restriction_list; - std::vector barrier_node_list; - std::vector traffic_light_list; - std::vector edge_list; - - unsigned requested_num_threads; - boost::filesystem::path config_file_path; - boost::filesystem::path input_path; - boost::filesystem::path restrictions_path; - boost::filesystem::path preinfo_path; - boost::filesystem::path profile_path; - - std::string node_filename; - std::string edge_out; - std::string info_out; - std::string geometry_filename; - std::string graph_out; - std::string rtree_nodes_path; - std::string rtree_leafs_path; + ContractorConfig config; }; #endif // PROCESSING_CHAIN_HPP diff --git a/data_structures/dynamic_graph.hpp b/data_structures/dynamic_graph.hpp index 43e6c3aca..b18204c61 100644 --- a/data_structures/dynamic_graph.hpp +++ b/data_structures/dynamic_graph.hpp @@ -85,6 +85,9 @@ template class DynamicGraph edge_list.resize(number_of_nodes); } + /** + * Constructs a DynamicGraph from a list of edges sorted by source node id. + */ template DynamicGraph(const NodeIterator nodes, const ContainerT &graph) { number_of_nodes = nodes; diff --git a/data_structures/node_based_graph.hpp b/data_structures/node_based_graph.hpp index bd71362c1..eda1752f1 100644 --- a/data_structures/node_based_graph.hpp +++ b/data_structures/node_based_graph.hpp @@ -125,8 +125,15 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector::max(); - // remove parallel edges + // remove parallel edges and set current distance values while (i < edges_list.size() && edges_list[i].source == source && edges_list[i].target == target) { @@ -167,17 +174,23 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector(forward_edge.data.distance) != std::numeric_limits::max()) { forward_edge.data.backward = true; + BOOST_ASSERT(edge_count < i); edges_list[edge_count++] = forward_edge; } } else { // insert seperate edges + // this case can only happen if we merged a bi-directional edge with a directional + // edge above, this incrementing i and making it safe to overwrite the next element + // as well if (static_cast(forward_edge.data.distance) != std::numeric_limits::max()) { + BOOST_ASSERT(edge_count < i); edges_list[edge_count++] = forward_edge; } if (static_cast(reverse_edge.data.distance) != std::numeric_limits::max()) { + BOOST_ASSERT(edge_count < i); edges_list[edge_count++] = reverse_edge; } } @@ -186,9 +199,8 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector( + return std::make_shared( static_cast(number_of_nodes), edges_list); - return graph; } #endif // NODE_BASED_GRAPH_HPP diff --git a/data_structures/restriction_map.hpp b/data_structures/restriction_map.hpp index 27a6698db..e2a89a008 100644 --- a/data_structures/restriction_map.hpp +++ b/data_structures/restriction_map.hpp @@ -91,6 +91,7 @@ template <> struct hash class RestrictionMap { public: + RestrictionMap() = default; RestrictionMap(const std::vector &restriction_list); // Replace end v with w in each turn restriction containing u as via node @@ -98,7 +99,7 @@ class RestrictionMap void FixupArrivingTurnRestriction(const NodeID node_u, const NodeID node_v, const NodeID node_w, - const std::shared_ptr &graph) + const GraphT &graph) { BOOST_ASSERT(node_u != SPECIAL_NODEID); BOOST_ASSERT(node_v != SPECIAL_NODEID); @@ -112,9 +113,9 @@ class RestrictionMap // find all potential start edges. It is more efficent to get a (small) list // of potential start edges than iterating over all buckets std::vector predecessors; - for (const EdgeID current_edge_id : graph->GetAdjacentEdgeRange(node_u)) + for (const EdgeID current_edge_id : graph.GetAdjacentEdgeRange(node_u)) { - const NodeID target = graph->GetTarget(current_edge_id); + const NodeID target = graph.GetTarget(current_edge_id); if (node_v != target) { predecessors.push_back(target); diff --git a/data_structures/static_rtree.hpp b/data_structures/static_rtree.hpp index 48ba20120..fa5607198 100644 --- a/data_structures/static_rtree.hpp +++ b/data_structures/static_rtree.hpp @@ -344,7 +344,7 @@ class StaticRTree StaticRTree(const StaticRTree &) = delete; // Construct a packed Hilbert-R-Tree with Kamel-Faloutsos algorithm [1] - explicit StaticRTree(std::vector &input_data_vector, + explicit StaticRTree(const std::vector &input_data_vector, const std::string tree_node_filename, const std::string leaf_node_filename, const std::vector &coordinate_list) diff --git a/extract.cpp b/extract.cpp index a14762983..5f88cc301 100644 --- a/extract.cpp +++ b/extract.cpp @@ -73,7 +73,7 @@ int main(int argc, char *argv[]) << " not found!"; return 1; } - return extractor().run(extractor_config); + return extractor(extractor_config).run(); } catch (const std::exception &e) { diff --git a/extractor/extractor.cpp b/extractor/extractor.cpp index fce0f8898..90fad4eec 100644 --- a/extractor/extractor.cpp +++ b/extractor/extractor.cpp @@ -82,7 +82,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * .restrictions : Turn restrictions that are used my osrm-prepare to construct the edge-expanded graph * */ -int extractor::run(const ExtractorConfig &extractor_config) +int extractor::run() { try { @@ -91,20 +91,20 @@ int extractor::run(const ExtractorConfig &extractor_config) const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); const auto number_of_threads = - std::min(recommended_num_threads, extractor_config.requested_num_threads); + std::min(recommended_num_threads, config.requested_num_threads); tbb::task_scheduler_init init(number_of_threads); - SimpleLogger().Write() << "Input file: " << extractor_config.input_path.filename().string(); - SimpleLogger().Write() << "Profile: " << extractor_config.profile_path.filename().string(); + SimpleLogger().Write() << "Input file: " << config.input_path.filename().string(); + SimpleLogger().Write() << "Profile: " << config.profile_path.filename().string(); SimpleLogger().Write() << "Threads: " << number_of_threads; // setup scripting environment - ScriptingEnvironment scripting_environment(extractor_config.profile_path.string().c_str()); + ScriptingEnvironment scripting_environment(config.profile_path.string().c_str()); ExtractionContainers extraction_containers; auto extractor_callbacks = osrm::make_unique(extraction_containers); - const osmium::io::File input_file(extractor_config.input_path.string()); + const osmium::io::File input_file(config.input_path.string()); osmium::io::Reader reader(input_file); const osmium::io::Header header = reader.header(); @@ -131,7 +131,7 @@ int extractor::run(const ExtractorConfig &extractor_config) } SimpleLogger().Write() << "timestamp: " << timestamp; - boost::filesystem::ofstream timestamp_out(extractor_config.timestamp_file_name); + boost::filesystem::ofstream timestamp_out(config.timestamp_file_name); timestamp_out.write(timestamp.c_str(), timestamp.length()); timestamp_out.close(); @@ -236,12 +236,12 @@ int extractor::run(const ExtractorConfig &extractor_config) return 1; } - extraction_containers.PrepareData(extractor_config.output_file_name, - extractor_config.restriction_file_name); + extraction_containers.PrepareData(config.output_file_name, + config.restriction_file_name); TIMER_STOP(extracting); SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s"; SimpleLogger().Write() << "To prepare the data for routing, run: " - << "./osrm-prepare " << extractor_config.output_file_name + << "./osrm-prepare " << config.output_file_name << std::endl; } catch (std::exception &e) diff --git a/extractor/extractor.hpp b/extractor/extractor.hpp index 8ea56c3cb..72defca05 100644 --- a/extractor/extractor.hpp +++ b/extractor/extractor.hpp @@ -30,8 +30,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "extractor_options.hpp" -struct extractor +class extractor { - int run(const ExtractorConfig &extractor_config); +public: + extractor(const ExtractorConfig &extractor_config) + : config(extractor_config) {} + int run(); +private: + ExtractorConfig config; }; #endif /* EXTRACTOR_HPP */ diff --git a/features/options/prepare/help.feature b/features/options/prepare/help.feature index 49497ae85..dae522d56 100644 --- a/features/options/prepare/help.feature +++ b/features/options/prepare/help.feature @@ -17,7 +17,7 @@ Feature: osrm-prepare command line options: help And stdout should contain "--profile" And stdout should contain "--threads" And stdout should contain 15 lines - And it should exit with code 0 + And it should exit with code 1 Scenario: osrm-prepare - Help, short When I run "osrm-prepare -h" diff --git a/prepare.cpp b/prepare.cpp index af10df035..5de320463 100644 --- a/prepare.cpp +++ b/prepare.cpp @@ -26,10 +26,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "contractor/processing_chain.hpp" +#include "contractor/contractor_options.hpp" #include "util/simple_logger.hpp" #include +#include + #include #include @@ -37,21 +40,64 @@ int main(int argc, char *argv[]) { try { - return Prepare().Process(argc, argv); - } - catch (boost::program_options::too_many_positional_options_error &) - { - SimpleLogger().Write(logWARNING) << "Only one file can be specified"; - return 1; - } - catch (boost::program_options::error &e) - { - SimpleLogger().Write(logWARNING) << e.what(); - return 1; + LogPolicy::GetInstance().Unmute(); + ContractorConfig contractor_config; + + const return_code result = ContractorOptions::ParseArguments(argc, argv, contractor_config); + + if (return_code::fail == result) + { + return 1; + } + + if (return_code::exit == result) + { + return 0; + } + + ContractorOptions::GenerateOutputFilesNames(contractor_config); + + if (1 > contractor_config.requested_num_threads) + { + SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger"; + return 1; + } + + const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); + + if (recommended_num_threads != contractor_config.requested_num_threads) + { + SimpleLogger().Write(logWARNING) << "The recommended number of threads is " + << recommended_num_threads + << "! This setting may have performance side-effects."; + } + + if (!boost::filesystem::is_regular_file(contractor_config.osrm_input_path)) + { + SimpleLogger().Write(logWARNING) + << "Input file " << contractor_config.osrm_input_path.string() << " not found!"; + return 1; + } + + if (!boost::filesystem::is_regular_file(contractor_config.profile_path)) + { + SimpleLogger().Write(logWARNING) << "Profile " << contractor_config.profile_path.string() + << " not found!"; + return 1; + } + + SimpleLogger().Write() << "Input file: " << contractor_config.osrm_input_path.filename().string(); + SimpleLogger().Write() << "Restrictions file: " << contractor_config.restrictions_path.filename().string(); + SimpleLogger().Write() << "Profile: " << contractor_config.profile_path.filename().string(); + SimpleLogger().Write() << "Threads: " << contractor_config.requested_num_threads; + + tbb::task_scheduler_init init(contractor_config.requested_num_threads); + + return Prepare(contractor_config).Run(); } catch (const std::exception &e) { - SimpleLogger().Write(logWARNING) << "Exception occured: " << e.what() << std::endl; + SimpleLogger().Write(logWARNING) << "[exception] " << e.what(); return 1; } } diff --git a/tools/components.cpp b/tools/components.cpp index 7ac64f8d9..f35d7196d 100644 --- a/tools/components.cpp +++ b/tools/components.cpp @@ -76,12 +76,73 @@ void DeleteFileIfExists(const std::string &file_name) } } +void LoadRestrictions(const char* path, + std::unique_ptr> ext_to_int_id_map, + std::vector& restriction_list) +{ + std::ifstream input_stream(path, std::ios::binary); + if (!input_stream.is_open()) + { + throw osrm::exception("Cannot open restriction file"); + } + loadRestrictionsFromFile(input_stream, *ext_to_int_id_map, restriction_list); +} + +std::size_t LoadGraph(const char* path, + std::vector& coordinate_list, + std::vector& barrier_node_list, + std::unordered_map& ext_to_int_id_map, + std::vector& graph_edge_list) +{ + std::ifstream input_stream(path, std::ifstream::in | std::ifstream::binary); + if (!input_stream.is_open()) + { + throw osrm::exception("Cannot open osrm file"); + } + + // load graph data + std::vector edge_list; + std::vector traffic_light_node_list; + + auto number_of_nodes = loadNodesFromFile(input_stream, barrier_node_list, + traffic_light_node_list, + coordinate_list, + ext_to_int_id_map); + + auto number_of_edges = loadEdgesFromFile(input_stream, ext_to_int_id_map, edge_list); + + traffic_light_node_list.clear(); + traffic_light_node_list.shrink_to_fit(); + + // Building an node-based graph + for (const auto &input_edge : edge_list) + { + if (input_edge.source == input_edge.target) + { + continue; + } + + if (input_edge.forward) + { + graph_edge_list.emplace_back(input_edge.source, input_edge.target, + (std::max)(input_edge.weight, 1), input_edge.name_id); + } + if (input_edge.backward) + { + graph_edge_list.emplace_back(input_edge.target, input_edge.source, + (std::max)(input_edge.weight, 1), input_edge.name_id); + } + } + + return number_of_nodes; +} + int main(int argc, char *argv[]) { std::vector coordinate_list; std::vector restriction_list; - std::vector bollard_node_list; - std::vector traffic_lights_list; + std::vector barrier_node_list; + auto ext_to_int_id_map = osrm::make_unique>(); LogPolicy::GetInstance().Unmute(); try @@ -95,83 +156,10 @@ int main(int argc, char *argv[]) } SimpleLogger().Write() << "Using restrictions from file: " << argv[2]; - std::ifstream restriction_ifstream(argv[2], std::ios::binary); - const FingerPrint fingerprint_orig; - FingerPrint fingerprint_loaded; - restriction_ifstream.read(reinterpret_cast(&fingerprint_loaded), - sizeof(FingerPrint)); - // check fingerprint and warn if necessary - if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) - { - SimpleLogger().Write(logWARNING) << argv[2] << " was prepared with a different build. " - "Reprocess to get rid of this warning."; - } - - if (!restriction_ifstream.good()) - { - throw osrm::exception("Could not access files"); - } - uint32_t usable_restrictions = 0; - restriction_ifstream.read(reinterpret_cast(&usable_restrictions), sizeof(uint32_t)); - restriction_list.resize(usable_restrictions); - - // load restrictions - if (usable_restrictions > 0) - { - restriction_ifstream.read(reinterpret_cast(&restriction_list[0]), - usable_restrictions * sizeof(TurnRestriction)); - } - restriction_ifstream.close(); - - std::ifstream input_stream(argv[1], std::ifstream::in | std::ifstream::binary); - if (!input_stream.is_open()) - { - throw osrm::exception("Cannot open osrm file"); - } - - // load graph data - std::vector edge_list; - const NodeID number_of_nodes = - readBinaryOSRMGraphFromStream(input_stream, edge_list, bollard_node_list, - traffic_lights_list, &coordinate_list, restriction_list); - input_stream.close(); - - BOOST_ASSERT_MSG(restriction_list.size() == usable_restrictions, - "size of restriction_list changed"); - - SimpleLogger().Write() << restriction_list.size() << " restrictions, " - << bollard_node_list.size() << " bollard nodes, " - << traffic_lights_list.size() << " traffic lights"; - - traffic_lights_list.clear(); - traffic_lights_list.shrink_to_fit(); - - // Building an node-based graph std::vector graph_edge_list; -// DeallocatingVector graph_edge_list; - for (const auto &input_edge : edge_list) - { - if (input_edge.source == input_edge.target) - { - continue; - } - - if (input_edge.forward) - { - graph_edge_list.emplace_back(input_edge.source, input_edge.target, - (std::max)(input_edge.weight, 1), input_edge.name_id); - } - if (input_edge.backward) - { - graph_edge_list.emplace_back(input_edge.target, input_edge.source, - (std::max)(input_edge.weight, 1), input_edge.name_id); - } - } - edge_list.clear(); - edge_list.shrink_to_fit(); - BOOST_ASSERT_MSG(0 == edge_list.size() && 0 == edge_list.capacity(), - "input edge vector not properly deallocated"); + auto number_of_nodes = LoadGraph(argv[1], coordinate_list, barrier_node_list, *ext_to_int_id_map, graph_edge_list); + LoadRestrictions(argv[2], std::move(ext_to_int_id_map), restriction_list); tbb::parallel_sort(graph_edge_list.begin(), graph_edge_list.end()); const auto graph = std::make_shared(number_of_nodes, graph_edge_list); @@ -181,9 +169,8 @@ int main(int argc, char *argv[]) SimpleLogger().Write() << "Starting SCC graph traversal"; RestrictionMap restriction_map(restriction_list); - auto tarjan = osrm::make_unique>(graph, - restriction_map, - bollard_node_list); + auto tarjan = osrm::make_unique>(graph, restriction_map, + barrier_node_list); tarjan->run(); SimpleLogger().Write() << "identified: " << tarjan->get_number_of_components() << " many components"; diff --git a/tools/graph_compare.cpp b/tools/graph_compare.cpp deleted file mode 100644 index 8adc06e78..000000000 --- a/tools/graph_compare.cpp +++ /dev/null @@ -1,199 +0,0 @@ -#include "../data_structures/dynamic_graph.hpp" -#include "../data_structures/import_edge.hpp" -#include "../data_structures/query_node.hpp" -#include "../data_structures/restriction.hpp" -#include "../data_structures/static_graph.hpp" -#include "../util/fingerprint.hpp" -#include "../util/graph_loader.hpp" -#include "../util/integer_range.hpp" -#include "../util/make_unique.hpp" -#include "../util/osrm_exception.hpp" -#include "../util/simple_logger.hpp" - -#include "../typedefs.h" - -#include -#include - -struct TarjanEdgeData -{ - TarjanEdgeData() : distance(INVALID_EDGE_WEIGHT), name_id(INVALID_NAMEID) {} - TarjanEdgeData(unsigned distance, unsigned name_id) : distance(distance), name_id(name_id) {} - unsigned distance; - unsigned name_id; -}; - -using StaticTestGraph = StaticGraph; -using DynamicTestGraph = StaticGraph; -using StaticEdge = StaticTestGraph::InputEdge; -using DynamicEdge = DynamicTestGraph::InputEdge; - -int main(int argc, char *argv[]) -{ - std::vector coordinate_list; - std::vector restriction_list; - std::vector bollard_node_list; - std::vector traffic_lights_list; - - LogPolicy::GetInstance().Unmute(); - try - { - // enable logging - if (argc < 3) - { - SimpleLogger().Write(logWARNING) << "usage:\n" << argv[0] - << " "; - return -1; - } - - SimpleLogger().Write() << "Using restrictions from file: " << argv[2]; - std::ifstream restriction_ifstream(argv[2], std::ios::binary); - const FingerPrint fingerprint_orig; - FingerPrint fingerprint_loaded; - restriction_ifstream.read(reinterpret_cast(&fingerprint_loaded), - sizeof(FingerPrint)); - - // check fingerprint and warn if necessary - if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) - { - SimpleLogger().Write(logWARNING) << argv[2] << " was prepared with a different build. " - "Reprocess to get rid of this warning."; - } - - if (!restriction_ifstream.good()) - { - throw osrm::exception("Could not access files"); - } - uint32_t usable_restrictions = 0; - restriction_ifstream.read(reinterpret_cast(&usable_restrictions), sizeof(uint32_t)); - restriction_list.resize(usable_restrictions); - - // load restrictions - if (usable_restrictions > 0) - { - restriction_ifstream.read(reinterpret_cast(&restriction_list[0]), - usable_restrictions * sizeof(TurnRestriction)); - } - restriction_ifstream.close(); - - std::ifstream input_stream(argv[1], std::ifstream::in | std::ifstream::binary); - if (!input_stream.is_open()) - { - throw osrm::exception("Cannot open osrm file"); - } - - // load graph data - std::vector edge_list; - const NodeID number_of_nodes = - readBinaryOSRMGraphFromStream(input_stream, edge_list, bollard_node_list, - traffic_lights_list, &coordinate_list, restriction_list); - input_stream.close(); - - BOOST_ASSERT_MSG(restriction_list.size() == usable_restrictions, - "size of restriction_list changed"); - - SimpleLogger().Write() << restriction_list.size() << " restrictions, " - << bollard_node_list.size() << " bollard nodes, " - << traffic_lights_list.size() << " traffic lights"; - - traffic_lights_list.clear(); - traffic_lights_list.shrink_to_fit(); - - // Building an node-based graph - std::vector static_graph_edge_list; - std::vector dynamic_graph_edge_list; - for (const auto &input_edge : edge_list) - { - if (input_edge.source == input_edge.target) - { - continue; - } - - if (input_edge.forward) - { - static_graph_edge_list.emplace_back(input_edge.source, input_edge.target, - (std::max)(input_edge.weight, 1), - input_edge.name_id); - dynamic_graph_edge_list.emplace_back(input_edge.source, input_edge.target, - (std::max)(input_edge.weight, 1), - input_edge.name_id); - } - if (input_edge.backward) - { - dynamic_graph_edge_list.emplace_back(input_edge.target, input_edge.source, - (std::max)(input_edge.weight, 1), - input_edge.name_id); - static_graph_edge_list.emplace_back(input_edge.target, input_edge.source, - (std::max)(input_edge.weight, 1), - input_edge.name_id); - } - } - edge_list.clear(); - edge_list.shrink_to_fit(); - BOOST_ASSERT_MSG(0 == edge_list.size() && 0 == edge_list.capacity(), - "input edge vector not properly deallocated"); - - tbb::parallel_sort(static_graph_edge_list.begin(), static_graph_edge_list.end()); - tbb::parallel_sort(dynamic_graph_edge_list.begin(), dynamic_graph_edge_list.end()); - - auto static_graph = - osrm::make_unique(number_of_nodes, static_graph_edge_list); - auto dynamic_graph = - osrm::make_unique(number_of_nodes, dynamic_graph_edge_list); - - SimpleLogger().Write() << "Starting static/dynamic graph comparison"; - - BOOST_ASSERT(static_graph->GetNumberOfNodes() == dynamic_graph->GetNumberOfNodes()); - BOOST_ASSERT(static_graph->GetNumberOfEdges() == dynamic_graph->GetNumberOfEdges()); - for (const auto node : osrm::irange(0u, static_graph->GetNumberOfNodes())) - { - const auto static_range = static_graph->GetAdjacentEdgeRange(node); - const auto dynamic_range = dynamic_graph->GetAdjacentEdgeRange(node); - SimpleLogger().Write() << "checking node " << node << "/" - << static_graph->GetNumberOfNodes(); - - BOOST_ASSERT(static_range.size() == dynamic_range.size()); - const auto static_begin = static_graph->BeginEdges(node); - const auto dynamic_begin = dynamic_graph->BeginEdges(node); - - // check raw interface - for (const auto i : osrm::irange(0u, static_range.size())) - { - const auto static_target = static_graph->GetTarget(static_begin + i); - const auto dynamic_target = dynamic_graph->GetTarget(dynamic_begin + i); - BOOST_ASSERT(static_target == dynamic_target); - - const auto static_data = static_graph->GetEdgeData(static_begin + i); - const auto dynamic_data = dynamic_graph->GetEdgeData(dynamic_begin + i); - - BOOST_ASSERT(static_data.distance == dynamic_data.distance); - BOOST_ASSERT(static_data.name_id == dynamic_data.name_id); - } - - // check range interface - std::vector static_target_ids, dynamic_target_ids; - std::vector static_edge_data, dynamic_edge_data; - for (const auto static_id : static_range) - { - static_target_ids.push_back(static_graph->GetTarget(static_id)); - static_edge_data.push_back(static_graph->GetEdgeData(static_id)); - } - for (const auto dynamic_id : dynamic_range) - { - dynamic_target_ids.push_back(dynamic_graph->GetTarget(dynamic_id)); - dynamic_edge_data.push_back(dynamic_graph->GetEdgeData(dynamic_id)); - } - BOOST_ASSERT(static_target_ids.size() == dynamic_target_ids.size()); - BOOST_ASSERT(std::equal(std::begin(static_target_ids), std::end(static_target_ids), - std::begin(dynamic_target_ids))); - } - - SimpleLogger().Write() << "Graph comparison finished successfully"; - } - catch (const std::exception &e) - { - SimpleLogger().Write(logWARNING) << "[exception] " << e.what(); - } - - return 0; -} diff --git a/util/graph_loader.hpp b/util/graph_loader.hpp index d741334ba..3779f0b9e 100644 --- a/util/graph_loader.hpp +++ b/util/graph_loader.hpp @@ -52,269 +52,34 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -template -NodeID read_undirected_osrm_stream(std::istream &input_stream, - std::vector &edge_list, - std::vector &coordinate_list) +/** + * Reads the .restrictions file and loads it to a vector. + * The since the restrictions reference nodes using their external node id, + * we need to renumber it to the new internal id. +*/ +unsigned loadRestrictionsFromFile(std::istream& input_stream, + const std::unordered_map& ext_to_int_id_map, + std::vector& restriction_list) { const FingerPrint fingerprint_orig; FingerPrint fingerprint_loaded; - input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(FingerPrint)); - - if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) + unsigned number_of_usable_restrictions = 0; + input_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint)); + if (!fingerprint_loaded.TestPrepare(fingerprint_orig)) { - SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n" + SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n" "Reprocess to get rid of this warning."; } - std::unordered_map ext_to_int_id_map; - - NodeID n; - input_stream.read(reinterpret_cast(&n), sizeof(NodeID)); - SimpleLogger().Write() << "Importing n = " << n << " nodes "; - - ExternalMemoryNode current_node; - for (NodeID i = 0; i < n; ++i) + input_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned)); + restriction_list.resize(number_of_usable_restrictions); + if (number_of_usable_restrictions > 0) { - input_stream.read(reinterpret_cast(¤t_node), sizeof(ExternalMemoryNode)); - coordinate_list.emplace_back(current_node.lat, current_node.lon); - ext_to_int_id_map.emplace(current_node.node_id, i); - // if (current_node.barrier) - // { - // barrier_node_list.emplace_back(i); - // } - // if (current_node.traffic_lights) - // { - // traffic_light_node_list.emplace_back(i); - // } + input_stream.read((char *) restriction_list.data(), + number_of_usable_restrictions * sizeof(TurnRestriction)); } - // tighten vector sizes - // barrier_node_list.shrink_to_fit(); - // traffic_light_node_list.shrink_to_fit(); - - // renumber nodes in turn restrictions - // for (TurnRestriction ¤t_restriction : restriction_list) - // { - // auto internal_id_iter = ext_to_int_id_map.find(current_restriction.from.node); - // if (internal_id_iter == ext_to_int_id_map.end()) - // { - // SimpleLogger().Write(logDEBUG) << "Unmapped from node " << - // current_restriction.from.node - // << " of restriction"; - // continue; - // } - // current_restriction.from.node = internal_id_iter->second; - - // internal_id_iter = ext_to_int_id_map.find(current_restriction.via.node); - // if (internal_id_iter == ext_to_int_id_map.end()) - // { - // SimpleLogger().Write(logDEBUG) << "Unmapped via node " << - // current_restriction.via.node - // << " of restriction"; - // continue; - // } - - // current_restriction.via.node = internal_id_iter->second; - - // internal_id_iter = ext_to_int_id_map.find(current_restriction.to.node); - // if (internal_id_iter == ext_to_int_id_map.end()) - // { - // SimpleLogger().Write(logDEBUG) << "Unmapped to node " << current_restriction.to.node - // << " of restriction"; - // continue; - // } - // current_restriction.to.node = internal_id_iter->second; - // } - - EdgeWeight weight; - NodeID source, target; - unsigned nameID; - int length; - short dir; // direction (0 = open, 1 = forward, 2+ = open) - bool is_roundabout, ignore_in_grid, is_access_restricted, is_split; - TravelMode travel_mode; - - EdgeID m; - input_stream.read(reinterpret_cast(&m), sizeof(unsigned)); - edge_list.reserve(m); - SimpleLogger().Write() << " and " << m << " edges "; - - for (EdgeID i = 0; i < m; ++i) - { - input_stream.read(reinterpret_cast(&source), sizeof(unsigned)); - input_stream.read(reinterpret_cast(&target), sizeof(unsigned)); - input_stream.read(reinterpret_cast(&length), sizeof(int)); - input_stream.read(reinterpret_cast(&dir), sizeof(short)); - input_stream.read(reinterpret_cast(&weight), sizeof(int)); - input_stream.read(reinterpret_cast(&nameID), sizeof(unsigned)); - input_stream.read(reinterpret_cast(&is_roundabout), sizeof(bool)); - input_stream.read(reinterpret_cast(&ignore_in_grid), sizeof(bool)); - input_stream.read(reinterpret_cast(&is_access_restricted), sizeof(bool)); - input_stream.read(reinterpret_cast(&travel_mode), sizeof(TravelMode)); - input_stream.read(reinterpret_cast(&is_split), sizeof(bool)); - - BOOST_ASSERT_MSG(length > 0, "loaded null length edge"); - BOOST_ASSERT_MSG(weight > 0, "loaded null weight"); - BOOST_ASSERT_MSG(0 <= dir && dir <= 2, "loaded bogus direction"); - - // bool forward = true; - // bool backward = true; - // if (1 == dir) - // { - // backward = false; - // } - // if (2 == dir) - // { - // forward = false; - // } - - // translate the external NodeIDs to internal IDs - auto internal_id_iter = ext_to_int_id_map.find(source); - if (ext_to_int_id_map.find(source) == ext_to_int_id_map.end()) - { -#ifndef NDEBUG - SimpleLogger().Write(logWARNING) << " unresolved source NodeID: " << source; -#endif - continue; - } - source = internal_id_iter->second; - internal_id_iter = ext_to_int_id_map.find(target); - if (ext_to_int_id_map.find(target) == ext_to_int_id_map.end()) - { -#ifndef NDEBUG - SimpleLogger().Write(logWARNING) << "unresolved target NodeID : " << target; -#endif - continue; - } - target = internal_id_iter->second; - BOOST_ASSERT_MSG(source != SPECIAL_NODEID && target != SPECIAL_NODEID, - "nonexisting source or target"); - - if (source > target) - { - std::swap(source, target); - // std::swap(forward, backward); - } - - edge_list.emplace_back(source, target); - } - ext_to_int_id_map.clear(); - - tbb::parallel_sort(edge_list.begin(), edge_list.end()); - - // for (unsigned i = 1; i < edge_list.size(); ++i) - // { - // if ((edge_list[i - 1].target == edge_list[i].target) && - // (edge_list[i - 1].source == edge_list[i].source)) - // { - // const bool edge_flags_equivalent = (edge_list[i - 1].forward == edge_list[i].forward) - // && - // (edge_list[i - 1].backward == - // edge_list[i].backward); - // const bool edge_flags_are_superset1 = - // (edge_list[i - 1].forward && edge_list[i - 1].backward) && - // (edge_list[i].forward != edge_list[i].backward); - // const bool edge_flags_are_superset_2 = - // (edge_list[i].forward && edge_list[i].backward) && - // (edge_list[i - 1].forward != edge_list[i - 1].backward); - - // if (edge_flags_equivalent) - // { - // edge_list[i].weight = std::min(edge_list[i - 1].weight, edge_list[i].weight); - // edge_list[i - 1].source = SPECIAL_NODEID; - // } - // else if (edge_flags_are_superset1) - // { - // if (edge_list[i - 1].weight <= edge_list[i].weight) - // { - // // edge i-1 is smaller and goes in both directions. Throw away the other edge - // edge_list[i].source = SPECIAL_NODEID; - // } - // else - // { - // // edge i-1 is open in both directions, but edge i is smaller in one - // direction. - // // Close edge i-1 in this direction - // edge_list[i - 1].forward = !edge_list[i].forward; - // edge_list[i - 1].backward = !edge_list[i].backward; - // } - // } - // else if (edge_flags_are_superset_2) - // { - // if (edge_list[i - 1].weight <= edge_list[i].weight) - // { - // // edge i-1 is smaller for one direction. edge i is open in both. close edge - // i - // // in the other direction - // edge_list[i].forward = !edge_list[i - 1].forward; - // edge_list[i].backward = !edge_list[i - 1].backward; - // } - // else - // { - // // edge i is smaller and goes in both direction. Throw away edge i-1 - // edge_list[i - 1].source = SPECIAL_NODEID; - // } - // } - // } - // } - // const auto new_end_iter = - // std::remove_if(edge_list.begin(), edge_list.end(), [](const EdgeT &edge) - // { - // return edge.source == SPECIAL_NODEID || edge.target == SPECIAL_NODEID; - // }); - // edge_list.erase(new_end_iter, edge_list.end()); // remove excess candidates. - // edge_list.shrink_to_fit(); - SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges"; - return n; -} - -template -NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream, - std::vector &edge_list, - std::vector &barrier_node_list, - std::vector &traffic_light_node_list, - std::vector *int_to_ext_node_id_map, - std::vector &restriction_list) -{ - const FingerPrint fingerprint_orig; - FingerPrint fingerprint_loaded; - input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(FingerPrint)); - - if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) - { - SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n" - "Reprocess to get rid of this warning."; - } - - std::unordered_map ext_to_int_id_map; - - NodeID n; - input_stream.read(reinterpret_cast(&n), sizeof(NodeID)); - SimpleLogger().Write() << "Importing n = " << n << " nodes "; - - ExternalMemoryNode current_node; - for (NodeID i = 0; i < n; ++i) - { - input_stream.read(reinterpret_cast(¤t_node), sizeof(ExternalMemoryNode)); - int_to_ext_node_id_map->emplace_back(current_node.lat, current_node.lon, - current_node.node_id); - ext_to_int_id_map.emplace(current_node.node_id, i); - if (current_node.barrier) - { - barrier_node_list.emplace_back(i); - } - if (current_node.traffic_lights) - { - traffic_light_node_list.emplace_back(i); - } - } - - // tighten vector sizes - barrier_node_list.shrink_to_fit(); - traffic_light_node_list.shrink_to_fit(); - - // renumber nodes in turn restrictions + // renumber ids referenced in restrictions for (TurnRestriction ¤t_restriction : restriction_list) { auto internal_id_iter = ext_to_int_id_map.find(current_restriction.from.node); @@ -346,6 +111,70 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream, current_restriction.to.node = internal_id_iter->second; } + return number_of_usable_restrictions; +} + + +/** + * Reads the beginning of an .osrm file and produces: + * - list of barrier nodes + * - list of traffic lights + * - index to original node id map + * - original node id to index map + */ +NodeID loadNodesFromFile(std::istream &input_stream, + std::vector &barrier_node_list, + std::vector &traffic_light_node_list, + std::vector &int_to_ext_node_id_map, + std::unordered_map &ext_to_int_id_map) +{ + const FingerPrint fingerprint_orig; + FingerPrint fingerprint_loaded; + input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(FingerPrint)); + + if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig)) + { + SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n" + "Reprocess to get rid of this warning."; + } + + NodeID n; + input_stream.read(reinterpret_cast(&n), sizeof(NodeID)); + SimpleLogger().Write() << "Importing n = " << n << " nodes "; + + ExternalMemoryNode current_node; + for (NodeID i = 0; i < n; ++i) + { + input_stream.read(reinterpret_cast(¤t_node), sizeof(ExternalMemoryNode)); + int_to_ext_node_id_map.emplace_back(current_node.lat, current_node.lon, + current_node.node_id); + ext_to_int_id_map.emplace(current_node.node_id, i); + if (current_node.barrier) + { + barrier_node_list.emplace_back(i); + } + if (current_node.traffic_lights) + { + traffic_light_node_list.emplace_back(i); + } + } + + // tighten vector sizes + barrier_node_list.shrink_to_fit(); + traffic_light_node_list.shrink_to_fit(); + + return n; +} + +/** + * Reads a .osrm file and produces the edges. Edges reference nodes in the old + * OSM based format, we need to renumber it here. + */ +template +NodeID loadEdgesFromFile(std::istream &input_stream, + const std::unordered_map &ext_to_int_id_map, + std::vector &edge_list) +{ EdgeWeight weight; NodeID source, target; unsigned nameID; @@ -419,10 +248,10 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream, edge_list.emplace_back(source, target, nameID, weight, forward, backward, is_roundabout, ignore_in_grid, is_access_restricted, travel_mode, is_split); } - ext_to_int_id_map.clear(); tbb::parallel_sort(edge_list.begin(), edge_list.end()); + // Removes multi-edges between nodes for (unsigned i = 1; i < edge_list.size(); ++i) { if ((edge_list[i - 1].target == edge_list[i].target) && @@ -482,7 +311,8 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream, edge_list.erase(new_end_iter, edge_list.end()); // remove excess candidates. edge_list.shrink_to_fit(); SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges"; - return n; + + return m; } template