diff --git a/Contractor/Prepare.cpp b/Contractor/Prepare.cpp new file mode 100644 index 000000000..19a5bbc67 --- /dev/null +++ b/Contractor/Prepare.cpp @@ -0,0 +1,570 @@ +/* + +Copyright (c) 2013, Project OSRM, Dennis Luxen, others +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "Prepare.h" +#include "../Algorithms/IteratorBasedCRC32.h" +#include "../Contractor/Contractor.h" +#include "../Contractor/EdgeBasedGraphFactory.h" +#include "../DataStructures/BinaryHeap.h" +#include "../DataStructures/DeallocatingVector.h" +#include "../DataStructures/StaticGraph.h" +#include "../DataStructures/StaticRTree.h" +#include "../DataStructures/RestrictionMap.h" +#include "../Util/GitDescription.h" +#include "../Util/LuaUtil.h" +#include "../Util/OSRMException.h" + +#include "../Util/SimpleLogger.h" +#include "../Util/StringUtil.h" +#include "../Util/TimingUtil.h" +#include "../typedefs.h" + +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include + + +Prepare::Prepare() : + requested_num_threads(1) +{ +} + +Prepare::~Prepare() +{ +} + +int Prepare::Process(int argc, char *argv[]) +{ + LogPolicy::GetInstance().Unmute(); + TIMER_START(preparing); + TIMER_START(expansion); + + if(!ParseArguments(argc, argv)) + return 0; + + if (!boost::filesystem::is_regular_file(input_path)) + { + SimpleLogger().Write(logWARNING) << "Input file " << input_path.string() + << " not found!"; + return 1; + } + + if (!boost::filesystem::is_regular_file(profile_path)) + { + SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string() + << " not found!"; + return 1; + } + + if (1 > requested_num_threads) + { + SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger"; + return 1; + } + + const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); + + SimpleLogger().Write() << "Input file: " << input_path.filename().string(); + SimpleLogger().Write() << "Restrictions file: " << restrictions_path.filename().string(); + SimpleLogger().Write() << "Profile: " << profile_path.filename().string(); + SimpleLogger().Write() << "Threads: " << requested_num_threads; + if (recommended_num_threads != requested_num_threads) + { + SimpleLogger().Write(logWARNING) << "The recommended number of threads is " + << recommended_num_threads + << "! This setting may have performance side-effects."; + } + + tbb::task_scheduler_init init(requested_num_threads); + + LogPolicy::GetInstance().Unmute(); + + + FingerPrint fingerprint_orig; + CheckRestrictionsFile(fingerprint_orig); + + boost::filesystem::ifstream in; + in.open(input_path, std::ios::in | std::ios::binary); + + node_filename = input_path.string() + ".nodes"; + edge_out = input_path.string() + ".edges"; + geometry_filename = input_path.string() + ".geometry"; + graph_out = input_path.string() + ".hsgr"; + rtree_nodes_path = input_path.string() + ".ramIndex"; + rtree_leafs_path = input_path.string() + ".fileIndex"; + + /*** Setup Scripting Environment ***/ + // Create a new lua state + lua_State *lua_state = luaL_newstate(); + + // Connect LuaBind to this lua state + luabind::open(lua_state); + + EdgeBasedGraphFactory::SpeedProfileProperties speed_profile; + + if(!SetupScriptingEnvironment(lua_state, speed_profile)) + return 1; + + #ifdef WIN32 + #pragma message ("Memory consumption on Windows can be higher due to memory alignment") + #else + static_assert(sizeof(ImportEdge) == 20, + "changing ImportEdge type has influence on memory consumption!"); + #endif + NodeID number_of_node_based_nodes = + readBinaryOSRMGraphFromStream(in, + edge_list, + barrier_node_list, + traffic_light_list, + &internal_to_external_node_map, + restriction_list); + in.close(); + + if (edge_list.empty()) + { + SimpleLogger().Write(logWARNING) << "The input data is empty, exiting."; + return 1; + } + + SimpleLogger().Write() << restriction_list.size() << " restrictions, " + << barrier_node_list.size() << " bollard nodes, " + << traffic_light_list.size() << " traffic lights"; + + std::vector node_based_edge_list; + unsigned number_of_edge_based_nodes = 0; + DeallocatingVector edge_based_edge_list; + + // init node_based_edge_list, edge_based_edge_list by edgeList + BuildEdgeExpandedGraph(lua_state, number_of_node_based_nodes, number_of_edge_based_nodes, + node_based_edge_list, edge_based_edge_list, speed_profile); + lua_close(lua_state); + + TIMER_STOP(expansion); + + BuildRTree(node_based_edge_list); + + IteratorbasedCRC32> crc32; + const unsigned node_based_edge_list_CRC32 = + crc32(node_based_edge_list.begin(), node_based_edge_list.end()); + node_based_edge_list.clear(); + node_based_edge_list.shrink_to_fit(); + SimpleLogger().Write() << "CRC32: " << node_based_edge_list_CRC32; + + WriteNodeMapping(); + + /*** + * Contracting the edge-expanded graph + */ + + SimpleLogger().Write() << "initializing contractor"; + Contractor *contractor = new Contractor(number_of_edge_based_nodes, edge_based_edge_list); + + TIMER_START(contraction); + contractor->Run(); + TIMER_STOP(contraction); + + SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec"; + + DeallocatingVector contracted_edge_list; + contractor->GetEdges(contracted_edge_list); + delete contractor; + + /*** + * Sorting contracted edges in a way that the static query graph can read some in in-place. + */ + + std::sort(contracted_edge_list.begin(), contracted_edge_list.end()); + unsigned max_used_node_id = 0; + unsigned contracted_edge_count = contracted_edge_list.size(); + SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count + << " edges"; + + boost::filesystem::ofstream hsgr_output_stream(graph_out, std::ios::binary); + hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint)); + for (const QueryEdge &edge : contracted_edge_list) + { + BOOST_ASSERT(UINT_MAX != edge.source); + BOOST_ASSERT(UINT_MAX != edge.target); + + max_used_node_id = std::max(max_used_node_id, edge.source); + max_used_node_id = std::max(max_used_node_id, edge.target); + } + SimpleLogger().Write(logDEBUG) << "input graph has " << number_of_edge_based_nodes + << " nodes"; + SimpleLogger().Write(logDEBUG) << "contracted graph has " << max_used_node_id << " nodes"; + max_used_node_id += 1; + + std::vector::NodeArrayEntry> node_array; + node_array.resize(number_of_edge_based_nodes + 1); + + SimpleLogger().Write() << "Building node array"; + StaticGraph::EdgeIterator edge = 0; + StaticGraph::EdgeIterator position = 0; + StaticGraph::EdgeIterator last_edge = edge; + + // initializing 'first_edge'-field of nodes: + for (auto node = 0u; node < max_used_node_id; ++node) + { + last_edge = edge; + while ((edge < contracted_edge_count) && (contracted_edge_list[edge].source == node)) + { + ++edge; + } + node_array[node].first_edge = position; //=edge + position += edge - last_edge; // remove + } + + for (unsigned sentinel_counter = max_used_node_id; sentinel_counter < node_array.size(); + ++sentinel_counter) + { + // sentinel element, guarded against underflow + node_array[sentinel_counter].first_edge = contracted_edge_count; + } + + SimpleLogger().Write() << "Serializing node array"; + + const unsigned node_array_size = node_array.size(); + // serialize crc32, aka checksum + hsgr_output_stream.write((char *)&node_based_edge_list_CRC32, sizeof(unsigned)); + // serialize number of nodes + hsgr_output_stream.write((char *)&node_array_size, sizeof(unsigned)); + // serialize number of edges + hsgr_output_stream.write((char *)&contracted_edge_count, sizeof(unsigned)); + // serialize all nodes + if (node_array_size > 0) + { + hsgr_output_stream.write((char *)&node_array[0], + sizeof(StaticGraph::NodeArrayEntry) * node_array_size); + } + // serialize all edges + + SimpleLogger().Write() << "Building edge array"; + edge = 0; + int number_of_used_edges = 0; + + StaticGraph::EdgeArrayEntry current_edge; + for (unsigned edge = 0; edge < contracted_edge_list.size(); ++edge) + { + // no eigen loops + BOOST_ASSERT(contracted_edge_list[edge].source != contracted_edge_list[edge].target); + current_edge.target = contracted_edge_list[edge].target; + current_edge.data = contracted_edge_list[edge].data; + + // every target needs to be valid + BOOST_ASSERT(current_edge.target < max_used_node_id); +#ifndef NDEBUG + if (current_edge.data.distance <= 0) + { + SimpleLogger().Write(logWARNING) + << "Edge: " << edge << ",source: " << contracted_edge_list[edge].source + << ", target: " << contracted_edge_list[edge].target + << ", dist: " << current_edge.data.distance; + + SimpleLogger().Write(logWARNING) << "Failed at adjacency list of node " + << contracted_edge_list[edge].source << "/" + << node_array.size() - 1; + return 1; + } +#endif + hsgr_output_stream.write((char *)¤t_edge, + sizeof(StaticGraph::EdgeArrayEntry)); + + ++number_of_used_edges; + } + hsgr_output_stream.close(); + + TIMER_STOP(preparing); + + SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds"; + SimpleLogger().Write() << "Expansion : " + << (number_of_node_based_nodes / TIMER_SEC(expansion)) + << " nodes/sec and " + << (number_of_edge_based_nodes / TIMER_SEC(expansion)) + << " edges/sec"; + + SimpleLogger().Write() << "Contraction: " + << (number_of_edge_based_nodes / TIMER_SEC(contraction)) + << " nodes/sec and " + << number_of_used_edges / TIMER_SEC(contraction) + << " edges/sec"; + + node_array.clear(); + SimpleLogger().Write() << "finished preprocessing"; + + return 0; +} + +/** + \brief Parses command line arguments + \param argc count of arguments + \param argv array of arguments + \param result [out] value for exit return value + \return true if everything is ok, false if need to terminate execution +*/ +bool Prepare::ParseArguments(int argc, char *argv[]) +{ + // declare a group of options that will be allowed only on command line + boost::program_options::options_description generic_options("Options"); + generic_options.add_options()("version,v", "Show version")("help,h", + "Show this help message")( + "config,c", + boost::program_options::value(&config_file_path) + ->default_value("contractor.ini"), + "Path to a configuration file."); + + // declare a group of options that will be allowed both on command line and in config file + boost::program_options::options_description config_options("Configuration"); + config_options.add_options()( + "restrictions,r", + boost::program_options::value(&restrictions_path), + "Restrictions file in .osrm.restrictions format")( + "profile,p", + boost::program_options::value(&profile_path) + ->default_value("profile.lua"), + "Path to LUA routing profile")( + "threads,t", + boost::program_options::value(&requested_num_threads)->default_value(tbb::task_scheduler_init::default_num_threads()), + "Number of threads to use"); + + // hidden options, will be allowed both on command line and in config file, but will not be + // shown to the user + boost::program_options::options_description hidden_options("Hidden options"); + hidden_options.add_options()( + "input,i", + boost::program_options::value(&input_path), + "Input file in .osm, .osm.bz2 or .osm.pbf format"); + + // positional option + boost::program_options::positional_options_description positional_options; + positional_options.add("input", 1); + + // combine above options for parsing + boost::program_options::options_description cmdline_options; + cmdline_options.add(generic_options).add(config_options).add(hidden_options); + + boost::program_options::options_description config_file_options; + config_file_options.add(config_options).add(hidden_options); + + boost::program_options::options_description visible_options( + "Usage: " + boost::filesystem::basename(argv[0]) + " [options]"); + visible_options.add(generic_options).add(config_options); + + // parse command line options + boost::program_options::variables_map option_variables; + boost::program_options::store(boost::program_options::command_line_parser(argc, argv) + .options(cmdline_options) + .positional(positional_options) + .run(), + option_variables); + + if (option_variables.count("version")) + { + SimpleLogger().Write() << g_GIT_DESCRIPTION; + return false; + } + + if (option_variables.count("help")) + { + SimpleLogger().Write() << "\n" << visible_options; + return false; + } + + boost::program_options::notify(option_variables); + + if (!option_variables.count("restrictions")) + { + restrictions_path = std::string(input_path.string() + ".restrictions"); + } + + if (!option_variables.count("input")) + { + SimpleLogger().Write() << "\n" << visible_options; + return false; + } + + return true; +} + +/** + \brief Loads and checks file UUIDs +*/ +void Prepare::CheckRestrictionsFile(FingerPrint &fingerprint_orig) +{ + boost::filesystem::ifstream restriction_stream(restrictions_path, std::ios::binary); + FingerPrint fingerprint_loaded; + unsigned number_of_usable_restrictions = 0; + restriction_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint)); + if (!fingerprint_loaded.TestPrepare(fingerprint_orig)) + { + SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n" + "Reprocess to get rid of this warning."; + } + + restriction_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned)); + restriction_list.resize(number_of_usable_restrictions); + if (number_of_usable_restrictions > 0) + { + restriction_stream.read((char *)&(restriction_list[0]), + number_of_usable_restrictions * sizeof(TurnRestriction)); + } + restriction_stream.close(); +} + +/** + \brief Setups scripting environment (lua-scripting) + Also initializes speed profile. +*/ +bool Prepare::SetupScriptingEnvironment(lua_State *lua_state, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile) +{ + // open utility libraries string library; + luaL_openlibs(lua_state); + + // adjust lua load path + luaAddScriptFolderToLoadPath(lua_state, profile_path.string().c_str()); + + // Now call our function in a lua script + if (0 != luaL_dofile(lua_state, profile_path.string().c_str())) + { + std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; + return false; + } + + if (0 != luaL_dostring(lua_state, "return traffic_signal_penalty\n")) + { + std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; + return false; + } + speed_profile.trafficSignalPenalty = 10 * lua_tointeger(lua_state, -1); + SimpleLogger().Write(logDEBUG) + << "traffic_signal_penalty: " << speed_profile.trafficSignalPenalty; + + if (0 != luaL_dostring(lua_state, "return u_turn_penalty\n")) + { + std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; + return false; + } + speed_profile.uTurnPenalty = 10 * lua_tointeger(lua_state, -1); + + speed_profile.has_turn_penalty_function = lua_function_exists(lua_state, "turn_function"); + + return true; +} + +/** + \brief Building an edge-expanded graph from node-based input and turn restrictions +*/ +void Prepare::BuildEdgeExpandedGraph(lua_State *lua_state, + NodeID number_of_node_based_nodes, + unsigned &number_of_edge_based_nodes, + std::vector &node_based_edge_list, + DeallocatingVector &edge_based_edge_list, + EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile) +{ + SimpleLogger().Write() << "Generating edge-expanded graph representation"; + std::shared_ptr node_based_graph = + NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list); + std::unique_ptr restriction_map = + std::unique_ptr(new RestrictionMap(node_based_graph, restriction_list)); + EdgeBasedGraphFactory *edge_based_graph_factor = + new EdgeBasedGraphFactory(node_based_graph, + std::move(restriction_map), + barrier_node_list, + traffic_light_list, + internal_to_external_node_map, + speed_profile); + edge_list.clear(); + edge_list.shrink_to_fit(); + + edge_based_graph_factor->Run(edge_out, geometry_filename, lua_state); + + restriction_list.clear(); + restriction_list.shrink_to_fit(); + barrier_node_list.clear(); + barrier_node_list.shrink_to_fit(); + traffic_light_list.clear(); + traffic_light_list.shrink_to_fit(); + + number_of_edge_based_nodes = edge_based_graph_factor->GetNumberOfEdgeBasedNodes(); + BOOST_ASSERT(number_of_edge_based_nodes != std::numeric_limits::max()); + #ifndef WIN32 + static_assert(sizeof(EdgeBasedEdge) == 16, + "changing ImportEdge type has influence on memory consumption!"); + #endif + + edge_based_graph_factor->GetEdgeBasedEdges(edge_based_edge_list); + edge_based_graph_factor->GetEdgeBasedNodes(node_based_edge_list); + delete edge_based_graph_factor; + + // TODO actually use scoping: Split this up in subfunctions + node_based_graph.reset(); +} + +/** + \brief Writing info on original (node-based) nodes + */ +void Prepare::WriteNodeMapping() +{ + SimpleLogger().Write() << "writing node map ..."; + boost::filesystem::ofstream node_stream(node_filename, std::ios::binary); + const unsigned size_of_mapping = internal_to_external_node_map.size(); + node_stream.write((char *)&size_of_mapping, sizeof(unsigned)); + if (size_of_mapping > 0) + { + node_stream.write((char *)&(internal_to_external_node_map[0]), + size_of_mapping * sizeof(NodeInfo)); + } + node_stream.close(); + internal_to_external_node_map.clear(); + internal_to_external_node_map.shrink_to_fit(); +} + +/** + \brief Building grid-like nearest-neighbor data structure + + Saves info to files: '.ramIndex' and '.fileIndex'. + */ +void Prepare::BuildRTree(std::vector &node_based_edge_list) +{ + SimpleLogger().Write() << "building r-tree ..."; + StaticRTree *rtree = + new StaticRTree(node_based_edge_list, + rtree_nodes_path.c_str(), + rtree_leafs_path.c_str(), + internal_to_external_node_map); + delete rtree; +} + diff --git a/Contractor/Prepare.h b/Contractor/Prepare.h new file mode 100644 index 000000000..ea23a7962 --- /dev/null +++ b/Contractor/Prepare.h @@ -0,0 +1,69 @@ +#ifndef PREPARE_H +#define PREPARE_H + +#include "EdgeBasedGraphFactory.h" +#include "../DataStructures/QueryEdge.h" +#include "../DataStructures/StaticGraph.h" +#include "../Util/GraphLoader.h" + +#include +#include + +#include + +#include + +/** + \brief class of 'prepare' utility. + */ +class Prepare +{ +public: + typedef QueryEdge::EdgeData EdgeData; + typedef DynamicGraph::InputEdge InputEdge; + typedef StaticGraph::InputEdge StaticEdge; + + + explicit Prepare(); + Prepare(const Prepare&) = delete; + ~Prepare(); + + int Process(int argc, char *argv[]); + +protected: + bool ParseArguments(int argc, char *argv[]); + void CheckRestrictionsFile(FingerPrint &fingerprint_orig); + bool SetupScriptingEnvironment(lua_State *myLuaState, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile); + void BuildEdgeExpandedGraph(lua_State *myLuaState, + NodeID nodeBasedNodeNumber, + unsigned &nodeBasedEdgesNumber, + std::vector &nodeBasedEdgeList, + DeallocatingVector &edgeBasedEdgeList, + EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile); + void WriteNodeMapping(); + void BuildRTree(std::vector &node_based_edge_list); + +private: + std::vector internal_to_external_node_map; + std::vector restriction_list; + std::vector barrier_node_list; + std::vector traffic_light_list; + std::vector edge_list; + + unsigned requested_num_threads; + boost::filesystem::path config_file_path; + boost::filesystem::path input_path; + boost::filesystem::path restrictions_path; + boost::filesystem::path preinfo_path; + boost::filesystem::path profile_path; + + std::string node_filename; + std::string edge_out; + std::string info_out; + std::string geometry_filename; + std::string graph_out; + std::string rtree_nodes_path; + std::string rtree_leafs_path; +}; + +#endif // PREPARE_H diff --git a/prepare.cpp b/prepare.cpp index 5565d9746..55f4c322b 100644 --- a/prepare.cpp +++ b/prepare.cpp @@ -25,489 +25,16 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "Algorithms/IteratorBasedCRC32.h" -#include "Contractor/Contractor.h" -#include "Contractor/EdgeBasedGraphFactory.h" -#include "DataStructures/BinaryHeap.h" -#include "DataStructures/DeallocatingVector.h" -#include "DataStructures/QueryEdge.h" -#include "DataStructures/StaticGraph.h" -#include "DataStructures/StaticRTree.h" -#include "DataStructures/RestrictionMap.h" -#include "Util/GitDescription.h" -#include "Util/GraphLoader.h" -#include "Util/LuaUtil.h" -#include "Util/OSRMException.h" +#include "Contractor/Prepare.h" -#include "Util/SimpleLogger.h" -#include "Util/StringUtil.h" -#include "Util/TimingUtil.h" -#include "typedefs.h" - -#include -#include - -#include - -#include -#include -#include -#include -#include - -#include -#include - -typedef QueryEdge::EdgeData EdgeData; -typedef DynamicGraph::InputEdge InputEdge; -typedef StaticGraph::InputEdge StaticEdge; - -std::vector internal_to_external_node_map; -std::vector restriction_list; -std::vector barrier_node_list; -std::vector traffic_light_list; -std::vector edge_list; +#include int main(int argc, char *argv[]) { try { - LogPolicy::GetInstance().Unmute(); - TIMER_START(preparing); - TIMER_START(expansion); - - boost::filesystem::path config_file_path, input_path, restrictions_path, profile_path; - unsigned int requested_num_threads; - - // declare a group of options that will be allowed only on command line - boost::program_options::options_description generic_options("Options"); - generic_options.add_options()("version,v", "Show version")("help,h", - "Show this help message")( - "config,c", - boost::program_options::value(&config_file_path) - ->default_value("contractor.ini"), - "Path to a configuration file."); - - // declare a group of options that will be allowed both on command line and in config file - boost::program_options::options_description config_options("Configuration"); - config_options.add_options()( - "restrictions,r", - boost::program_options::value(&restrictions_path), - "Restrictions file in .osrm.restrictions format")( - "profile,p", - boost::program_options::value(&profile_path) - ->default_value("profile.lua"), - "Path to LUA routing profile")( - "threads,t", - boost::program_options::value(&requested_num_threads)->default_value(tbb::task_scheduler_init::default_num_threads()), - "Number of threads to use"); - - // hidden options, will be allowed both on command line and in config file, but will not be - // shown to the user - boost::program_options::options_description hidden_options("Hidden options"); - hidden_options.add_options()( - "input,i", - boost::program_options::value(&input_path), - "Input file in .osm, .osm.bz2 or .osm.pbf format"); - - // positional option - boost::program_options::positional_options_description positional_options; - positional_options.add("input", 1); - - // combine above options for parsing - boost::program_options::options_description cmdline_options; - cmdline_options.add(generic_options).add(config_options).add(hidden_options); - - boost::program_options::options_description config_file_options; - config_file_options.add(config_options).add(hidden_options); - - boost::program_options::options_description visible_options( - "Usage: " + boost::filesystem::basename(argv[0]) + " [options]"); - visible_options.add(generic_options).add(config_options); - - // parse command line options - boost::program_options::variables_map option_variables; - boost::program_options::store(boost::program_options::command_line_parser(argc, argv) - .options(cmdline_options) - .positional(positional_options) - .run(), - option_variables); - - if (option_variables.count("version")) - { - SimpleLogger().Write() << g_GIT_DESCRIPTION; - return 0; - } - - if (option_variables.count("help")) - { - SimpleLogger().Write() << "\n" << visible_options; - return 0; - } - - boost::program_options::notify(option_variables); - - if (!option_variables.count("restrictions")) - { - restrictions_path = std::string(input_path.string() + ".restrictions"); - } - - if (!option_variables.count("input")) - { - SimpleLogger().Write() << "\n" << visible_options; - return 0; - } - - if (!boost::filesystem::is_regular_file(input_path)) - { - SimpleLogger().Write(logWARNING) << "Input file " << input_path.string() - << " not found!"; - return 1; - } - - if (!boost::filesystem::is_regular_file(profile_path)) - { - SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string() - << " not found!"; - return 1; - } - - if (1 > requested_num_threads) - { - SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger"; - return 1; - } - - const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); - - SimpleLogger().Write() << "Input file: " << input_path.filename().string(); - SimpleLogger().Write() << "Restrictions file: " << restrictions_path.filename().string(); - SimpleLogger().Write() << "Profile: " << profile_path.filename().string(); - SimpleLogger().Write() << "Threads: " << requested_num_threads; - if (recommended_num_threads != requested_num_threads) - { - SimpleLogger().Write(logWARNING) << "The recommended number of threads is " - << recommended_num_threads - << "! This setting may have performance side-effects."; - } - - tbb::task_scheduler_init init(requested_num_threads); - - LogPolicy::GetInstance().Unmute(); - boost::filesystem::ifstream restriction_stream(restrictions_path, std::ios::binary); - TurnRestriction restriction; - FingerPrint fingerprint_loaded, fingerprint_orig; - unsigned number_of_usable_restrictions = 0; - restriction_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint)); - if (!fingerprint_loaded.TestPrepare(fingerprint_orig)) - { - SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n" - "Reprocess to get rid of this warning."; - } - - restriction_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned)); - restriction_list.resize(number_of_usable_restrictions); - if (number_of_usable_restrictions > 0) - { - restriction_stream.read((char *)&(restriction_list[0]), - number_of_usable_restrictions * sizeof(TurnRestriction)); - } - restriction_stream.close(); - - boost::filesystem::ifstream in; - in.open(input_path, std::ios::in | std::ios::binary); - - const std::string node_filename = input_path.string() + ".nodes"; - const std::string edge_out = input_path.string() + ".edges"; - const std::string geometry_filename = input_path.string() + ".geometry"; - const std::string graphOut = input_path.string() + ".hsgr"; - const std::string rtree_nodes_path = input_path.string() + ".ramIndex"; - const std::string rtree_leafs_path = input_path.string() + ".fileIndex"; - - /*** Setup Scripting Environment ***/ - - // Create a new lua state - lua_State *lua_state = luaL_newstate(); - - // Connect LuaBind to this lua state - luabind::open(lua_state); - - // open utility libraries string library; - luaL_openlibs(lua_state); - - // adjust lua load path - luaAddScriptFolderToLoadPath(lua_state, profile_path.string().c_str()); - - // Now call our function in a lua script - if (0 != luaL_dofile(lua_state, profile_path.string().c_str())) - { - std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; - return 1; - } - - EdgeBasedGraphFactory::SpeedProfileProperties speed_profile; - - if (0 != luaL_dostring(lua_state, "return traffic_signal_penalty\n")) - { - std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; - return 1; - } - speed_profile.trafficSignalPenalty = 10 * lua_tointeger(lua_state, -1); - SimpleLogger().Write(logDEBUG) - << "traffic_signal_penalty: " << speed_profile.trafficSignalPenalty; - - if (0 != luaL_dostring(lua_state, "return u_turn_penalty\n")) - { - std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; - return 1; - } - speed_profile.uTurnPenalty = 10 * lua_tointeger(lua_state, -1); - - speed_profile.has_turn_penalty_function = lua_function_exists(lua_state, "turn_function"); - - #ifdef WIN32 - #pragma message ("Memory consumption on Windows can be higher due to memory alignment") - #else - static_assert(sizeof(ImportEdge) == 20, - "changing ImportEdge type has influence on memory consumption!"); - #endif - std::vector edge_list; - NodeID number_of_node_based_nodes = - readBinaryOSRMGraphFromStream(in, - edge_list, - barrier_node_list, - traffic_light_list, - &internal_to_external_node_map, - restriction_list); - in.close(); - - if (edge_list.empty()) - { - SimpleLogger().Write(logWARNING) << "The input data is empty, exiting."; - return 1; - } - - SimpleLogger().Write() << restriction_list.size() << " restrictions, " - << barrier_node_list.size() << " bollard nodes, " - << traffic_light_list.size() << " traffic lights"; - - /*** - * Building an edge-expanded graph from node-based input and turn restrictions - */ - - SimpleLogger().Write() << "Generating edge-expanded graph representation"; - std::shared_ptr node_based_graph = - NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list); - std::unique_ptr restriction_map = - std::unique_ptr(new RestrictionMap(node_based_graph, restriction_list)); - EdgeBasedGraphFactory *edge_based_graph_factor = - new EdgeBasedGraphFactory(node_based_graph, - std::move(restriction_map), - barrier_node_list, - traffic_light_list, - internal_to_external_node_map, - speed_profile); - edge_list.clear(); - edge_list.shrink_to_fit(); - - edge_based_graph_factor->Run(edge_out, geometry_filename, lua_state); - - restriction_list.clear(); - restriction_list.shrink_to_fit(); - barrier_node_list.clear(); - barrier_node_list.shrink_to_fit(); - traffic_light_list.clear(); - traffic_light_list.shrink_to_fit(); - - lua_close(lua_state); - - unsigned number_of_edge_based_nodes = edge_based_graph_factor->GetNumberOfEdgeBasedNodes(); - BOOST_ASSERT(number_of_edge_based_nodes != std::numeric_limits::max()); - DeallocatingVector edgeBasedEdgeList; - #ifndef WIN32 - static_assert(sizeof(EdgeBasedEdge) == 16, - "changing ImportEdge type has influence on memory consumption!"); - #endif - - edge_based_graph_factor->GetEdgeBasedEdges(edgeBasedEdgeList); - std::vector node_based_edge_list; - edge_based_graph_factor->GetEdgeBasedNodes(node_based_edge_list); - delete edge_based_graph_factor; - - // TODO actually use scoping: Split this up in subfunctions - node_based_graph.reset(); - - TIMER_STOP(expansion); - - // Building grid-like nearest-neighbor data structure - SimpleLogger().Write() << "building r-tree ..."; - StaticRTree *rtree = - new StaticRTree(node_based_edge_list, - rtree_nodes_path.c_str(), - rtree_leafs_path.c_str(), - internal_to_external_node_map); - delete rtree; - IteratorbasedCRC32> crc32; - const unsigned node_based_edge_list_CRC32 = - crc32(node_based_edge_list.begin(), node_based_edge_list.end()); - node_based_edge_list.clear(); - node_based_edge_list.shrink_to_fit(); - SimpleLogger().Write() << "CRC32: " << node_based_edge_list_CRC32; - - /*** - * Writing info on original (node-based) nodes - */ - - SimpleLogger().Write() << "writing node map ..."; - boost::filesystem::ofstream node_stream(node_filename, std::ios::binary); - const unsigned size_of_mapping = internal_to_external_node_map.size(); - node_stream.write((char *)&size_of_mapping, sizeof(unsigned)); - if (size_of_mapping > 0) - { - node_stream.write((char *)&(internal_to_external_node_map[0]), - size_of_mapping * sizeof(NodeInfo)); - } - node_stream.close(); - internal_to_external_node_map.clear(); - internal_to_external_node_map.shrink_to_fit(); - - /*** - * Contracting the edge-expanded graph - */ - - SimpleLogger().Write() << "initializing contractor"; - Contractor *contractor = new Contractor(number_of_edge_based_nodes, edgeBasedEdgeList); - - TIMER_START(contraction); - contractor->Run(); - TIMER_STOP(contraction); - - SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec"; - - DeallocatingVector contracted_edge_list; - contractor->GetEdges(contracted_edge_list); - delete contractor; - - /*** - * Sorting contracted edges in a way that the static query graph can read some in in-place. - */ - - std::sort(contracted_edge_list.begin(), contracted_edge_list.end()); - unsigned max_used_node_id = 0; - unsigned contracted_edge_count = contracted_edge_list.size(); - SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count - << " edges"; - - boost::filesystem::ofstream hsgr_output_stream(graphOut, std::ios::binary); - hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint)); - for (const QueryEdge &edge : contracted_edge_list) - { - BOOST_ASSERT(UINT_MAX != edge.source); - BOOST_ASSERT(UINT_MAX != edge.target); - - max_used_node_id = std::max(max_used_node_id, edge.source); - max_used_node_id = std::max(max_used_node_id, edge.target); - } - SimpleLogger().Write(logDEBUG) << "input graph has " << number_of_edge_based_nodes - << " nodes"; - SimpleLogger().Write(logDEBUG) << "contracted graph has " << max_used_node_id << " nodes"; - max_used_node_id += 1; - - std::vector::NodeArrayEntry> node_array; - node_array.resize(number_of_edge_based_nodes + 1); - - SimpleLogger().Write() << "Building node array"; - StaticGraph::EdgeIterator edge = 0; - StaticGraph::EdgeIterator position = 0; - StaticGraph::EdgeIterator last_edge = edge; - - for (auto node = 0; node < max_used_node_id; ++node) - { - last_edge = edge; - while ((edge < contracted_edge_count) && (contracted_edge_list[edge].source == node)) - { - ++edge; - } - node_array[node].first_edge = position; //=edge - position += edge - last_edge; // remove - } - - for (unsigned sentinel_counter = max_used_node_id; sentinel_counter < node_array.size(); - ++sentinel_counter) - { - // sentinel element, guarded against underflow - node_array[sentinel_counter].first_edge = contracted_edge_count; - } - - SimpleLogger().Write() << "Serializing node array"; - - const unsigned node_array_size = node_array.size(); - // serialize crc32, aka checksum - hsgr_output_stream.write((char *)&node_based_edge_list_CRC32, sizeof(unsigned)); - // serialize number of nodes - hsgr_output_stream.write((char *)&node_array_size, sizeof(unsigned)); - // serialize number of edges - hsgr_output_stream.write((char *)&contracted_edge_count, sizeof(unsigned)); - // serialize all nodes - if (node_array_size > 0) - { - hsgr_output_stream.write((char *)&node_array[0], - sizeof(StaticGraph::NodeArrayEntry) * node_array_size); - } - // serialize all edges - - SimpleLogger().Write() << "Building edge array"; - edge = 0; - int number_of_used_edges = 0; - - StaticGraph::EdgeArrayEntry current_edge; - for (unsigned edge = 0; edge < contracted_edge_list.size(); ++edge) - { - // no eigen loops - BOOST_ASSERT(contracted_edge_list[edge].source != contracted_edge_list[edge].target); - current_edge.target = contracted_edge_list[edge].target; - current_edge.data = contracted_edge_list[edge].data; - - // every target needs to be valid - BOOST_ASSERT(current_edge.target < max_used_node_id); -#ifndef NDEBUG - if (current_edge.data.distance <= 0) - { - SimpleLogger().Write(logWARNING) - << "Edge: " << edge << ",source: " << contracted_edge_list[edge].source - << ", target: " << contracted_edge_list[edge].target - << ", dist: " << current_edge.data.distance; - - SimpleLogger().Write(logWARNING) << "Failed at adjacency list of node " - << contracted_edge_list[edge].source << "/" - << node_array.size() - 1; - return 1; - } -#endif - hsgr_output_stream.write((char *)¤t_edge, - sizeof(StaticGraph::EdgeArrayEntry)); - - ++number_of_used_edges; - } - hsgr_output_stream.close(); - - TIMER_STOP(preparing); - - SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds"; - SimpleLogger().Write() << "Expansion : " - << (number_of_node_based_nodes / TIMER_SEC(expansion)) - << " nodes/sec and " - << (number_of_edge_based_nodes / TIMER_SEC(expansion)) - << " edges/sec"; - - SimpleLogger().Write() << "Contraction: " - << (number_of_edge_based_nodes / TIMER_SEC(contraction)) - << " nodes/sec and " - << number_of_used_edges / TIMER_SEC(contraction) - << " edges/sec"; - - node_array.clear(); - SimpleLogger().Write() << "finished preprocessing"; + Prepare prepare; + return prepare.Process(argc, argv); } catch (boost::program_options::too_many_positional_options_error &) {