diff --git a/CMakeLists.txt b/CMakeLists.txt index 21b46ba49..11562f895 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,7 +51,7 @@ configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/util/version.hpp.in ${CMAKE_CURRENT_SOURCE_DIR}/util/version.hpp ) -file(GLOB ExtractorGlob extractor/*.cpp) +file(GLOB ExtractorGlob extractor/*.cpp data_structures/hilbert_value.cpp) file(GLOB ImporterGlob data_structures/import_edge.cpp data_structures/external_memory_node.cpp data_structures/raster_source.cpp) add_library(IMPORT OBJECT ${ImporterGlob}) add_library(LOGGER OBJECT util/simple_logger.cpp) @@ -62,7 +62,7 @@ add_library(MERCATOR OBJECT util/mercator.cpp) add_library(ANGLE OBJECT util/compute_angle.cpp) set(ExtractorSources extract.cpp ${ExtractorGlob}) -add_executable(osrm-extract ${ExtractorSources} $ $ $ $ $ $) +add_executable(osrm-extract ${ExtractorSources} $ $ $ $ $ $ $ $ $ $) add_library(RESTRICTION OBJECT data_structures/restriction_map.cpp) add_library(COMPRESSEDEDGE OBJECT data_structures/compressed_edge_container.cpp) diff --git a/algorithms/graph_compressor.hpp b/algorithms/graph_compressor.hpp index 75405c0c9..8096a92d6 100644 --- a/algorithms/graph_compressor.hpp +++ b/algorithms/graph_compressor.hpp @@ -29,7 +29,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../typedefs.h" -#include "../contractor/speed_profile.hpp" +#include "../extractor/speed_profile.hpp" #include "../data_structures/node_based_graph.hpp" #include diff --git a/contractor/contractor_options.cpp b/contractor/contractor_options.cpp index f2aeb0b3e..73f932977 100644 --- a/contractor/contractor_options.cpp +++ b/contractor/contractor_options.cpp @@ -48,9 +48,6 @@ ContractorOptions::ParseArguments(int argc, char *argv[], ContractorConfig &cont // declare a group of options that will be allowed both on command line and in config file boost::program_options::options_description config_options("Configuration"); config_options.add_options()( - "restrictions,r", - boost::program_options::value(&contractor_config.restrictions_path), - "Restrictions file in .osrm.restrictions format")( "profile,p", boost::program_options::value(&contractor_config.profile_path) ->default_value("profile.lua"), "Path to LUA routing profile")( @@ -114,11 +111,6 @@ ContractorOptions::ParseArguments(int argc, char *argv[], ContractorConfig &cont boost::program_options::notify(option_variables); - if (!option_variables.count("restrictions")) - { - contractor_config.restrictions_path = contractor_config.osrm_input_path.string() + ".restrictions"; - } - if (!option_variables.count("input")) { SimpleLogger().Write() << "\n" << visible_options; @@ -130,11 +122,7 @@ ContractorOptions::ParseArguments(int argc, char *argv[], ContractorConfig &cont void ContractorOptions::GenerateOutputFilesNames(ContractorConfig &contractor_config) { - contractor_config.node_output_path = contractor_config.osrm_input_path.string() + ".nodes"; contractor_config.core_output_path = contractor_config.osrm_input_path.string() + ".core"; - contractor_config.edge_output_path = contractor_config.osrm_input_path.string() + ".edges"; - contractor_config.geometry_output_path = contractor_config.osrm_input_path.string() + ".geometry"; contractor_config.graph_output_path = contractor_config.osrm_input_path.string() + ".hsgr"; - contractor_config.rtree_nodes_output_path = contractor_config.osrm_input_path.string() + ".ramIndex"; - contractor_config.rtree_leafs_output_path = contractor_config.osrm_input_path.string() + ".fileIndex"; + contractor_config.edge_based_graph_path = contractor_config.osrm_input_path.string() + ".ebg"; } diff --git a/contractor/contractor_options.hpp b/contractor/contractor_options.hpp index 3836627d5..8ad2363e3 100644 --- a/contractor/contractor_options.hpp +++ b/contractor/contractor_options.hpp @@ -45,16 +45,11 @@ struct ContractorConfig boost::filesystem::path config_file_path; boost::filesystem::path osrm_input_path; - boost::filesystem::path restrictions_path; boost::filesystem::path profile_path; - std::string node_output_path; std::string core_output_path; - std::string edge_output_path; - std::string geometry_output_path; std::string graph_output_path; - std::string rtree_nodes_output_path; - std::string rtree_leafs_output_path; + std::string edge_based_graph_path; unsigned requested_num_threads; diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index bc9695b51..2aeae4837 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -28,14 +28,9 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "processing_chain.hpp" #include "contractor.hpp" -#include "../algorithms/graph_compressor.hpp" -#include "../algorithms/tarjan_scc.hpp" -#include "../algorithms/crc32_processor.hpp" -#include "../data_structures/compressed_edge_container.hpp" #include "../data_structures/deallocating_vector.hpp" -#include "../data_structures/static_rtree.hpp" -#include "../data_structures/restriction_map.hpp" +#include "../algorithms/crc32_processor.hpp" #include "../util/graph_loader.hpp" #include "../util/integer_range.hpp" #include "../util/lua_util.hpp" @@ -73,32 +68,11 @@ int Prepare::Run() // Create a new lua state - SimpleLogger().Write() << "Generating edge-expanded graph representation"; + SimpleLogger().Write() << "Loading edge-expanded graph representation"; - TIMER_START(expansion); - - std::vector node_based_edge_list; DeallocatingVector edge_based_edge_list; - std::vector internal_to_external_node_map; - auto graph_size = BuildEdgeExpandedGraph(internal_to_external_node_map, node_based_edge_list, - edge_based_edge_list); - auto number_of_node_based_nodes = graph_size.first; - auto max_edge_id = graph_size.second; - - TIMER_STOP(expansion); - - SimpleLogger().Write() << "building r-tree ..."; - TIMER_START(rtree); - - FindComponents(max_edge_id, edge_based_edge_list, node_based_edge_list); - - BuildRTree(node_based_edge_list, internal_to_external_node_map); - - TIMER_STOP(rtree); - - SimpleLogger().Write() << "writing node map ..."; - WriteNodeMapping(internal_to_external_node_map); + size_t max_edge_id = LoadEdgeExpandedGraph(config.edge_based_graph_path, edge_based_edge_list); // Contracting the edge-expanded graph @@ -111,16 +85,12 @@ int Prepare::Run() SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec"; std::size_t number_of_used_edges = - WriteContractedGraph(max_edge_id, node_based_edge_list, contracted_edge_list); + WriteContractedGraph(max_edge_id, contracted_edge_list); WriteCoreNodeMarker(std::move(is_core_node)); TIMER_STOP(preparing); SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds"; - SimpleLogger().Write() << "Expansion : " << (number_of_node_based_nodes / TIMER_SEC(expansion)) - << " nodes/sec and " << ((max_edge_id + 1) / TIMER_SEC(expansion)) - << " edges/sec"; - SimpleLogger().Write() << "Contraction: " << ((max_edge_id + 1) / TIMER_SEC(contraction)) << " nodes/sec and " << number_of_used_edges / TIMER_SEC(contraction) << " edges/sec"; @@ -130,81 +100,38 @@ int Prepare::Run() return 0; } -void Prepare::FindComponents(unsigned max_edge_id, - const DeallocatingVector &input_edge_list, - std::vector &input_nodes) const +std::size_t Prepare::LoadEdgeExpandedGraph( + std::string const & edge_based_graph_filename, + DeallocatingVector & edge_based_edge_list) { - struct UncontractedEdgeData - { - }; - struct InputEdge - { - unsigned source; - unsigned target; - UncontractedEdgeData data; + SimpleLogger().Write() << "Opening " << edge_based_graph_filename; + boost::filesystem::ifstream input_stream(edge_based_graph_filename, std::ios::in | std::ios::binary); - bool operator<(const InputEdge &rhs) const - { - return source < rhs.source || (source == rhs.source && target < rhs.target); - } + const FingerPrint fingerprint_valid = FingerPrint::GetValid(); + FingerPrint fingerprint_loaded; + input_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint)); + fingerprint_loaded.TestPrepare(fingerprint_valid); - bool operator==(const InputEdge &rhs) const - { - return source == rhs.source && target == rhs.target; - } - }; - using UncontractedGraph = StaticGraph; - std::vector edges; - edges.reserve(input_edge_list.size() * 2); + size_t number_of_edges = 0; + size_t max_edge_id = SPECIAL_EDGEID; + input_stream.read((char *)&number_of_edges, sizeof(size_t)); + input_stream.read((char *)&max_edge_id, sizeof(size_t)); - for (const auto &edge : input_edge_list) - { - BOOST_ASSERT_MSG(static_cast(std::max(edge.weight, 1)) > 0, - "edge distance < 1"); - if (edge.forward) - { - edges.push_back({edge.source, edge.target, {}}); - } + edge_based_edge_list.resize(number_of_edges); + SimpleLogger().Write() << "Reading " << number_of_edges << " edges from the edge based graph"; - if (edge.backward) - { - edges.push_back({edge.target, edge.source, {}}); - } - } - - // connect forward and backward nodes of each edge - for (const auto &node : input_nodes) - { - if (node.reverse_edge_based_node_id != SPECIAL_NODEID) - { - edges.push_back({node.forward_edge_based_node_id, node.reverse_edge_based_node_id, {}}); - edges.push_back({node.reverse_edge_based_node_id, node.forward_edge_based_node_id, {}}); - } - } - - tbb::parallel_sort(edges.begin(), edges.end()); - auto new_end = std::unique(edges.begin(), edges.end()); - edges.resize(new_end - edges.begin()); - - auto uncontractor_graph = std::make_shared(max_edge_id + 1, edges); - - TarjanSCC component_search( - std::const_pointer_cast(uncontractor_graph)); - component_search.run(); - - for (auto &node : input_nodes) - { - auto forward_component = component_search.get_component_id(node.forward_edge_based_node_id); - BOOST_ASSERT(node.reverse_edge_based_node_id == SPECIAL_EDGEID || - forward_component == - component_search.get_component_id(node.reverse_edge_based_node_id)); - - const unsigned component_size = component_search.get_component_size(forward_component); - const bool is_tiny_component = component_size < 1000; - node.component_id = is_tiny_component ? (1 + forward_component) : 0; + // TODO: can we read this in bulk? DeallocatingVector isn't necessarily + // all stored contiguously + for (;number_of_edges > 0; --number_of_edges) { + EdgeBasedEdge inbuffer; + input_stream.read((char *) &inbuffer, sizeof(EdgeBasedEdge)); + edge_based_edge_list.emplace_back(std::move(inbuffer)); } + SimpleLogger().Write() << "Done reading edges"; + return max_edge_id; } + void Prepare::WriteCoreNodeMarker(std::vector &&in_is_core_node) const { std::vector is_core_node(in_is_core_node); @@ -223,11 +150,8 @@ void Prepare::WriteCoreNodeMarker(std::vector &&in_is_core_node) const } std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, - const std::vector &node_based_edge_list, const DeallocatingVector &contracted_edge_list) { - const unsigned crc32_value = CalculateEdgeChecksum(node_based_edge_list); - // Sorting contracted edges in a way that the static query graph can read some in in-place. tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end()); const unsigned contracted_edge_count = contracted_edge_list.size(); @@ -283,9 +207,13 @@ std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, SimpleLogger().Write() << "Serializing node array"; + RangebasedCRC32 crc32_calculator; + const unsigned edges_crc32 = crc32_calculator(contracted_edge_list); + SimpleLogger().Write() << "Writing CRC32: " << edges_crc32; + const unsigned node_array_size = node_array.size(); // serialize crc32, aka checksum - hsgr_output_stream.write((char *)&crc32_value, sizeof(unsigned)); + hsgr_output_stream.write((char *)&edges_crc32, sizeof(unsigned)); // serialize number of nodes hsgr_output_stream.write((char *)&node_array_size, sizeof(unsigned)); // serialize number of edges @@ -334,165 +262,7 @@ std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, return number_of_used_edges; } -unsigned Prepare::CalculateEdgeChecksum(const std::vector &node_based_edge_list) -{ - RangebasedCRC32 crc32; - if (crc32.using_hardware()) - { - SimpleLogger().Write() << "using hardware based CRC32 computation"; - } - else - { - SimpleLogger().Write() << "using software based CRC32 computation"; - } - const unsigned crc32_value = crc32(node_based_edge_list); - SimpleLogger().Write() << "CRC32: " << crc32_value; - - return crc32_value; -} - -/** - \brief Setups scripting environment (lua-scripting) - Also initializes speed profile. -*/ -void Prepare::SetupScriptingEnvironment(lua_State *lua_state, SpeedProfileProperties &speed_profile) -{ - // open utility libraries string library; - luaL_openlibs(lua_state); - - // adjust lua load path - luaAddScriptFolderToLoadPath(lua_state, config.profile_path.string().c_str()); - - // Now call our function in a lua script - if (0 != luaL_dofile(lua_state, config.profile_path.string().c_str())) - { - std::stringstream msg; - msg << lua_tostring(lua_state, -1) << " occured in scripting block"; - throw osrm::exception(msg.str()); - } - - if (0 != luaL_dostring(lua_state, "return traffic_signal_penalty\n")) - { - std::stringstream msg; - msg << lua_tostring(lua_state, -1) << " occured in scripting block"; - throw osrm::exception(msg.str()); - } - speed_profile.traffic_signal_penalty = 10 * lua_tointeger(lua_state, -1); - SimpleLogger().Write(logDEBUG) - << "traffic_signal_penalty: " << speed_profile.traffic_signal_penalty; - - if (0 != luaL_dostring(lua_state, "return u_turn_penalty\n")) - { - std::stringstream msg; - msg << lua_tostring(lua_state, -1) << " occured in scripting block"; - throw osrm::exception(msg.str()); - } - - speed_profile.u_turn_penalty = 10 * lua_tointeger(lua_state, -1); - speed_profile.has_turn_penalty_function = lua_function_exists(lua_state, "turn_function"); -} - -/** - \brief Build load restrictions from .restriction file - */ -std::shared_ptr Prepare::LoadRestrictionMap() -{ - boost::filesystem::ifstream input_stream(config.restrictions_path, - std::ios::in | std::ios::binary); - - std::vector restriction_list; - loadRestrictionsFromFile(input_stream, restriction_list); - - SimpleLogger().Write() << " - " << restriction_list.size() << " restrictions."; - - return std::make_shared(restriction_list); -} - -/** - \brief Load node based graph from .osrm file - */ -std::shared_ptr -Prepare::LoadNodeBasedGraph(std::unordered_set &barrier_nodes, - std::unordered_set &traffic_lights, - std::vector &internal_to_external_node_map) -{ - std::vector edge_list; - - boost::filesystem::ifstream input_stream(config.osrm_input_path, - std::ios::in | std::ios::binary); - - std::vector barrier_list; - std::vector traffic_light_list; - NodeID number_of_node_based_nodes = loadNodesFromFile( - input_stream, barrier_list, traffic_light_list, internal_to_external_node_map); - - SimpleLogger().Write() << " - " << barrier_list.size() << " bollard nodes, " - << traffic_light_list.size() << " traffic lights"; - - // insert into unordered sets for fast lookup - barrier_nodes.insert(barrier_list.begin(), barrier_list.end()); - traffic_lights.insert(traffic_light_list.begin(), traffic_light_list.end()); - - barrier_list.clear(); - barrier_list.shrink_to_fit(); - traffic_light_list.clear(); - traffic_light_list.shrink_to_fit(); - - loadEdgesFromFile(input_stream, edge_list); - - if (edge_list.empty()) - { - SimpleLogger().Write(logWARNING) << "The input data is empty, exiting."; - return std::shared_ptr(); - } - - return NodeBasedDynamicGraphFromEdges(number_of_node_based_nodes, edge_list); -} - -/** - \brief Building an edge-expanded graph from node-based input and turn restrictions -*/ -std::pair -Prepare::BuildEdgeExpandedGraph(std::vector &internal_to_external_node_map, - std::vector &node_based_edge_list, - DeallocatingVector &edge_based_edge_list) -{ - lua_State *lua_state = luaL_newstate(); - luabind::open(lua_state); - - SpeedProfileProperties speed_profile; - SetupScriptingEnvironment(lua_state, speed_profile); - - std::unordered_set barrier_nodes; - std::unordered_set traffic_lights; - - auto restriction_map = LoadRestrictionMap(); - auto node_based_graph = - LoadNodeBasedGraph(barrier_nodes, traffic_lights, internal_to_external_node_map); - - CompressedEdgeContainer compressed_edge_container; - GraphCompressor graph_compressor(speed_profile); - graph_compressor.Compress(barrier_nodes, traffic_lights, *restriction_map, *node_based_graph, - compressed_edge_container); - - EdgeBasedGraphFactory edge_based_graph_factory( - node_based_graph, compressed_edge_container, barrier_nodes, traffic_lights, - std::const_pointer_cast(restriction_map), - internal_to_external_node_map, speed_profile); - - compressed_edge_container.SerializeInternalVector(config.geometry_output_path); - - edge_based_graph_factory.Run(config.edge_output_path, lua_state); - lua_close(lua_state); - - edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list); - edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list); - auto max_edge_id = edge_based_graph_factory.GetHighestEdgeID(); - - const std::size_t number_of_node_based_nodes = node_based_graph->GetNumberOfNodes(); - return std::make_pair(number_of_node_based_nodes, max_edge_id); -} /** \brief Build contracted graph. @@ -508,31 +278,4 @@ void Prepare::ContractGraph(const unsigned max_edge_id, contractor.GetCoreMarker(is_core_node); } -/** - \brief Writing info on original (node-based) nodes - */ -void Prepare::WriteNodeMapping(const std::vector &internal_to_external_node_map) -{ - boost::filesystem::ofstream node_stream(config.node_output_path, std::ios::binary); - const unsigned size_of_mapping = internal_to_external_node_map.size(); - node_stream.write((char *)&size_of_mapping, sizeof(unsigned)); - if (size_of_mapping > 0) - { - node_stream.write((char *)internal_to_external_node_map.data(), - size_of_mapping * sizeof(QueryNode)); - } - node_stream.close(); -} -/** - \brief Building rtree-based nearest-neighbor data structure - - Saves tree into '.ramIndex' and leaves into '.fileIndex'. - */ -void Prepare::BuildRTree(const std::vector &node_based_edge_list, - const std::vector &internal_to_external_node_map) -{ - StaticRTree(node_based_edge_list, config.rtree_nodes_output_path.c_str(), - config.rtree_leafs_output_path.c_str(), - internal_to_external_node_map); -} diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index 4e12baf6d..1743cc880 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -29,9 +29,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define PROCESSING_CHAIN_HPP #include "contractor_options.hpp" -#include "edge_based_graph_factory.hpp" #include "../data_structures/query_edge.hpp" #include "../data_structures/static_graph.hpp" +#include "../data_structures/deallocating_vector.hpp" +#include "../data_structures/node_based_graph.hpp" struct SpeedProfileProperties; struct EdgeBasedNode; @@ -48,8 +49,6 @@ class Prepare { public: using EdgeData = QueryEdge::EdgeData; - using InputEdge = DynamicGraph::InputEdge; - using StaticEdge = StaticGraph::InputEdge; explicit Prepare(ContractorConfig contractor_config) : config(std::move(contractor_config)) {} Prepare(const Prepare &) = delete; @@ -58,34 +57,21 @@ class Prepare int Run(); protected: - void SetupScriptingEnvironment(lua_State *myLuaState, SpeedProfileProperties &speed_profile); - unsigned CalculateEdgeChecksum(const std::vector &node_based_edge_list); void ContractGraph(const unsigned max_edge_id, DeallocatingVector &edge_based_edge_list, DeallocatingVector &contracted_edge_list, std::vector &is_core_node); void WriteCoreNodeMarker(std::vector &&is_core_node) const; std::size_t WriteContractedGraph(unsigned number_of_edge_based_nodes, - const std::vector &node_based_edge_list, const DeallocatingVector &contracted_edge_list); - std::shared_ptr LoadRestrictionMap(); - std::shared_ptr - LoadNodeBasedGraph(std::unordered_set &barrier_nodes, - std::unordered_set &traffic_lights, - std::vector &internal_to_external_node_map); - std::pair - BuildEdgeExpandedGraph(std::vector &internal_to_external_node_map, - std::vector &node_based_edge_list, - DeallocatingVector &edge_based_edge_list); - void WriteNodeMapping(const std::vector &internal_to_external_node_map); void FindComponents(unsigned max_edge_id, const DeallocatingVector &edges, std::vector &nodes) const; - void BuildRTree(const std::vector &node_based_edge_list, - const std::vector &internal_to_external_node_map); - private: ContractorConfig config; + std::size_t LoadEdgeExpandedGraph( + const std::string & edge_based_graph_path, + DeallocatingVector & edge_based_edge_list); }; #endif // PROCESSING_CHAIN_HPP diff --git a/contractor/edge_based_graph_factory.cpp b/extractor/edge_based_graph_factory.cpp similarity index 100% rename from contractor/edge_based_graph_factory.cpp rename to extractor/edge_based_graph_factory.cpp diff --git a/contractor/edge_based_graph_factory.hpp b/extractor/edge_based_graph_factory.hpp similarity index 100% rename from contractor/edge_based_graph_factory.hpp rename to extractor/edge_based_graph_factory.hpp diff --git a/extractor/extractor.cpp b/extractor/extractor.cpp index 6355bde0a..de6ad76da 100644 --- a/extractor/extractor.cpp +++ b/extractor/extractor.cpp @@ -39,9 +39,18 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../util/simple_logger.hpp" #include "../util/timing_util.hpp" #include "../util/lua_util.hpp" +#include "../util/graph_loader.hpp" #include "../typedefs.h" +#include "../data_structures/static_graph.hpp" +#include "../data_structures/static_rtree.hpp" +#include "../data_structures/restriction_map.hpp" +#include "../data_structures/compressed_edge_container.hpp" + +#include "../algorithms/tarjan_scc.hpp" +#include "../algorithms/crc32_processor.hpp" + #include #include #include @@ -253,14 +262,347 @@ int extractor::run() TIMER_STOP(extracting); SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s"; - SimpleLogger().Write() << "To prepare the data for routing, run: " - << "./osrm-prepare " << config.output_file_name - << std::endl; } - catch (std::exception &e) + catch (const std::exception &e) { SimpleLogger().Write(logWARNING) << e.what(); return 1; } + + try + { + // Transform the node-based graph that OSM is based on into an edge-based graph + // that is better for routing. Every edge becomes a node, and every valid + // movement (e.g. turn from A->B, and B->A) becomes an edge + // + // + // // Create a new lua state + + SimpleLogger().Write() << "Generating edge-expanded graph representation"; + + TIMER_START(expansion); + + std::vector node_based_edge_list; + DeallocatingVector edge_based_edge_list; + std::vector internal_to_external_node_map; + auto graph_size = + BuildEdgeExpandedGraph(internal_to_external_node_map, + node_based_edge_list, + edge_based_edge_list); + + auto number_of_node_based_nodes = graph_size.first; + auto max_edge_id = graph_size.second; + + TIMER_STOP(expansion); + + SimpleLogger().Write() << "building r-tree ..."; + TIMER_START(rtree); + + FindComponents(max_edge_id, edge_based_edge_list, node_based_edge_list); + + BuildRTree(node_based_edge_list, internal_to_external_node_map); + + TIMER_STOP(rtree); + + SimpleLogger().Write() << "writing node map ..."; + WriteNodeMapping(internal_to_external_node_map); + + WriteEdgeBasedGraph(config.edge_graph_output_path, max_edge_id, edge_based_edge_list); + + SimpleLogger().Write() << "Expansion : " << (number_of_node_based_nodes / TIMER_SEC(expansion)) + << " nodes/sec and " << ((max_edge_id + 1) / TIMER_SEC(expansion)) + << " edges/sec"; + SimpleLogger().Write() << "To prepare the data for routing, run: " + << "./osrm-prepare " << config.output_file_name + << std::endl; + } + catch (const std::exception &e) + { + SimpleLogger().Write(logWARNING) << e.what(); + return 1; + } + return 0; } + +/** + \brief Setups scripting environment (lua-scripting) + Also initializes speed profile. +*/ +void extractor::SetupScriptingEnvironment(lua_State *lua_state, SpeedProfileProperties &speed_profile) +{ + // open utility libraries string library; + luaL_openlibs(lua_state); + + // adjust lua load path + luaAddScriptFolderToLoadPath(lua_state, config.profile_path.string().c_str()); + + // Now call our function in a lua script + if (0 != luaL_dofile(lua_state, config.profile_path.string().c_str())) + { + std::stringstream msg; + msg << lua_tostring(lua_state, -1) << " occured in scripting block"; + throw osrm::exception(msg.str()); + } + + if (0 != luaL_dostring(lua_state, "return traffic_signal_penalty\n")) + { + std::stringstream msg; + msg << lua_tostring(lua_state, -1) << " occured in scripting block"; + throw osrm::exception(msg.str()); + } + speed_profile.traffic_signal_penalty = 10 * lua_tointeger(lua_state, -1); + SimpleLogger().Write(logDEBUG) + << "traffic_signal_penalty: " << speed_profile.traffic_signal_penalty; + + if (0 != luaL_dostring(lua_state, "return u_turn_penalty\n")) + { + std::stringstream msg; + msg << lua_tostring(lua_state, -1) << " occured in scripting block"; + throw osrm::exception(msg.str()); + } + + speed_profile.u_turn_penalty = 10 * lua_tointeger(lua_state, -1); + speed_profile.has_turn_penalty_function = lua_function_exists(lua_state, "turn_function"); +} + +void extractor::FindComponents(unsigned max_edge_id, + const DeallocatingVector &input_edge_list, + std::vector &input_nodes) const +{ + struct UncontractedEdgeData + { + }; + struct InputEdge + { + unsigned source; + unsigned target; + UncontractedEdgeData data; + + bool operator<(const InputEdge &rhs) const + { + return source < rhs.source || (source == rhs.source && target < rhs.target); + } + + bool operator==(const InputEdge &rhs) const + { + return source == rhs.source && target == rhs.target; + } + }; + using UncontractedGraph = StaticGraph; + std::vector edges; + edges.reserve(input_edge_list.size() * 2); + + for (const auto &edge : input_edge_list) + { + BOOST_ASSERT_MSG(static_cast(std::max(edge.weight, 1)) > 0, + "edge distance < 1"); + if (edge.forward) + { + edges.push_back({edge.source, edge.target, {}}); + } + + if (edge.backward) + { + edges.push_back({edge.target, edge.source, {}}); + } + } + + // connect forward and backward nodes of each edge + for (const auto &node : input_nodes) + { + if (node.reverse_edge_based_node_id != SPECIAL_NODEID) + { + edges.push_back({node.forward_edge_based_node_id, node.reverse_edge_based_node_id, {}}); + edges.push_back({node.reverse_edge_based_node_id, node.forward_edge_based_node_id, {}}); + } + } + + tbb::parallel_sort(edges.begin(), edges.end()); + auto new_end = std::unique(edges.begin(), edges.end()); + edges.resize(new_end - edges.begin()); + + auto uncontractor_graph = std::make_shared(max_edge_id + 1, edges); + + TarjanSCC component_search( + std::const_pointer_cast(uncontractor_graph)); + component_search.run(); + + for (auto &node : input_nodes) + { + auto forward_component = component_search.get_component_id(node.forward_edge_based_node_id); + BOOST_ASSERT(node.reverse_edge_based_node_id == SPECIAL_EDGEID || + forward_component == + component_search.get_component_id(node.reverse_edge_based_node_id)); + + const unsigned component_size = component_search.get_component_size(forward_component); + const bool is_tiny_component = component_size < 1000; + node.component_id = is_tiny_component ? (1 + forward_component) : 0; + } +} + +/** + \brief Build load restrictions from .restriction file + */ +std::shared_ptr extractor::LoadRestrictionMap() +{ + boost::filesystem::ifstream input_stream(config.restriction_file_name, + std::ios::in | std::ios::binary); + + std::vector restriction_list; + loadRestrictionsFromFile(input_stream, restriction_list); + + SimpleLogger().Write() << " - " << restriction_list.size() << " restrictions."; + + return std::make_shared(restriction_list); +} + +/** + \brief Load node based graph from .osrm file + */ +std::shared_ptr +extractor::LoadNodeBasedGraph(std::unordered_set &barrier_nodes, + std::unordered_set &traffic_lights, + std::vector &internal_to_external_node_map) +{ + std::vector edge_list; + + boost::filesystem::ifstream input_stream(config.output_file_name, + std::ios::in | std::ios::binary); + + std::vector barrier_list; + std::vector traffic_light_list; + NodeID number_of_node_based_nodes = loadNodesFromFile( + input_stream, barrier_list, traffic_light_list, internal_to_external_node_map); + + SimpleLogger().Write() << " - " << barrier_list.size() << " bollard nodes, " + << traffic_light_list.size() << " traffic lights"; + + // insert into unordered sets for fast lookup + barrier_nodes.insert(barrier_list.begin(), barrier_list.end()); + traffic_lights.insert(traffic_light_list.begin(), traffic_light_list.end()); + + barrier_list.clear(); + barrier_list.shrink_to_fit(); + traffic_light_list.clear(); + traffic_light_list.shrink_to_fit(); + + loadEdgesFromFile(input_stream, edge_list); + + if (edge_list.empty()) + { + SimpleLogger().Write(logWARNING) << "The input data is empty, exiting."; + return std::shared_ptr(); + } + + return NodeBasedDynamicGraphFromEdges(number_of_node_based_nodes, edge_list); +} + +/** + \brief Building an edge-expanded graph from node-based input and turn restrictions +*/ +std::pair +extractor::BuildEdgeExpandedGraph(std::vector &internal_to_external_node_map, + std::vector &node_based_edge_list, + DeallocatingVector &edge_based_edge_list) +{ + lua_State *lua_state = luaL_newstate(); + luabind::open(lua_state); + + SpeedProfileProperties speed_profile; + SetupScriptingEnvironment(lua_state, speed_profile); + + std::unordered_set barrier_nodes; + std::unordered_set traffic_lights; + + auto restriction_map = LoadRestrictionMap(); + auto node_based_graph = + LoadNodeBasedGraph(barrier_nodes, traffic_lights, internal_to_external_node_map); + + CompressedEdgeContainer compressed_edge_container; + GraphCompressor graph_compressor(speed_profile); + graph_compressor.Compress(barrier_nodes, traffic_lights, *restriction_map, *node_based_graph, + compressed_edge_container); + + EdgeBasedGraphFactory edge_based_graph_factory( + node_based_graph, compressed_edge_container, barrier_nodes, traffic_lights, + std::const_pointer_cast(restriction_map), + internal_to_external_node_map, speed_profile); + + + compressed_edge_container.SerializeInternalVector(config.geometry_output_path); + + edge_based_graph_factory.Run(config.edge_output_path, lua_state); + lua_close(lua_state); + + edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list); + edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list); + auto max_edge_id = edge_based_graph_factory.GetHighestEdgeID(); + + // danpat TODO: somewhere right around here, we will need to + // use the internal_to_external_node_map (which contains original OSM node ids) + // the edges from the compressed edge container + // and the edge-based-edges + + const std::size_t number_of_node_based_nodes = node_based_graph->GetNumberOfNodes(); + return std::make_pair(number_of_node_based_nodes, max_edge_id); +} + + +/** + \brief Writing info on original (node-based) nodes + */ +void extractor::WriteNodeMapping(const std::vector & internal_to_external_node_map) +{ + boost::filesystem::ofstream node_stream(config.node_output_path, std::ios::binary); + const unsigned size_of_mapping = internal_to_external_node_map.size(); + node_stream.write((char *)&size_of_mapping, sizeof(unsigned)); + if (size_of_mapping > 0) + { + node_stream.write((char *)internal_to_external_node_map.data(), + size_of_mapping * sizeof(QueryNode)); + } + node_stream.close(); +} + +/** + \brief Building rtree-based nearest-neighbor data structure + + Saves tree into '.ramIndex' and leaves into '.fileIndex'. + */ +void extractor::BuildRTree(const std::vector &node_based_edge_list, + const std::vector &internal_to_external_node_map) +{ + StaticRTree(node_based_edge_list, config.rtree_nodes_output_path.c_str(), + config.rtree_leafs_output_path.c_str(), + internal_to_external_node_map); +} + +void extractor::WriteEdgeBasedGraph(std::string const &output_file_filename, + size_t const max_edge_id, + DeallocatingVector const & edge_based_edge_list) +{ + + std::ofstream file_out_stream; + file_out_stream.open(output_file_filename.c_str(), std::ios::binary); + const FingerPrint fingerprint = FingerPrint::GetValid(); + file_out_stream.write((char *)&fingerprint, sizeof(FingerPrint)); + + std::cout << "[extractor] Writing edge-based-graph egdes ... " << std::flush; + TIMER_START(write_edges); + + size_t number_of_used_edges = edge_based_edge_list.size(); + file_out_stream.write((char *)&number_of_used_edges, sizeof(size_t)); + file_out_stream.write((char *)&max_edge_id, sizeof(size_t)); + + for (const auto& edge : edge_based_edge_list) { + file_out_stream.write((char *) &edge, sizeof(EdgeBasedEdge)); + } + + TIMER_STOP(write_edges); + std::cout << "ok, after " << TIMER_SEC(write_edges) << "s" << std::endl; + + SimpleLogger().Write() << "Processed " << number_of_used_edges << " edges"; + file_out_stream.close(); + +} diff --git a/extractor/extractor.hpp b/extractor/extractor.hpp index bc822413d..e100e5593 100644 --- a/extractor/extractor.hpp +++ b/extractor/extractor.hpp @@ -29,6 +29,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define EXTRACTOR_HPP #include "extractor_options.hpp" +#include "edge_based_graph_factory.hpp" +#include "../algorithms/graph_compressor.hpp" class extractor { @@ -36,6 +38,25 @@ public: extractor(ExtractorConfig extractor_config) : config(std::move(extractor_config)) {} int run(); private: - ExtractorConfig config; + ExtractorConfig config; + void SetupScriptingEnvironment(lua_State *myLuaState, + SpeedProfileProperties &speed_profile); + std::pair + BuildEdgeExpandedGraph(std::vector &internal_to_external_node_map, + std::vector &node_based_edge_list, + DeallocatingVector &edge_based_edge_list); + void WriteNodeMapping(const std::vector & internal_to_external_node_map); + void FindComponents(unsigned max_edge_id, const DeallocatingVector& edges, std::vector& nodes) const; + void BuildRTree(const std::vector &node_based_edge_list, + const std::vector &internal_to_external_node_map); + std::shared_ptr LoadRestrictionMap(); + std::shared_ptr + LoadNodeBasedGraph(std::unordered_set &barrier_nodes, + std::unordered_set &traffic_lights, + std::vector& internal_to_external_node_map); + + void WriteEdgeBasedGraph(std::string const &output_file_filename, + size_t const max_edge_id, + DeallocatingVector const & edge_based_edge_list); }; #endif /* EXTRACTOR_HPP */ diff --git a/extractor/extractor_options.cpp b/extractor/extractor_options.cpp index b0c5c9eff..86c927d80 100644 --- a/extractor/extractor_options.cpp +++ b/extractor/extractor_options.cpp @@ -42,6 +42,12 @@ ExtractorOptions::ParseArguments(int argc, char *argv[], ExtractorConfig &extrac // declare a group of options that will be allowed only on command line boost::program_options::options_description generic_options("Options"); generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message")( + /* + * TODO: re-enable this + "restrictions,r", + boost::program_options::value(&extractor_config.restrictions_path), + "Restrictions file in .osrm.restrictions format")( + */ "config,c", boost::program_options::value( &extractor_config.config_file_path)->default_value("extractor.ini"), "Path to a configuration file."); @@ -137,6 +143,12 @@ void ExtractorOptions::GenerateOutputFilesNames(ExtractorConfig &extractor_confi extractor_config.restriction_file_name = input_path.string(); extractor_config.names_file_name = input_path.string(); extractor_config.timestamp_file_name = input_path.string(); + extractor_config.geometry_output_path = input_path.string(); + extractor_config.edge_output_path = input_path.string(); + extractor_config.edge_graph_output_path = input_path.string(); + extractor_config.node_output_path = input_path.string(); + extractor_config.rtree_nodes_output_path = input_path.string(); + extractor_config.rtree_leafs_output_path = input_path.string(); std::string::size_type pos = extractor_config.output_file_name.find(".osm.bz2"); if (pos == std::string::npos) { @@ -159,6 +171,12 @@ void ExtractorOptions::GenerateOutputFilesNames(ExtractorConfig &extractor_confi extractor_config.restriction_file_name.append(".osrm.restrictions"); extractor_config.names_file_name.append(".osrm.names"); extractor_config.timestamp_file_name.append(".osrm.timestamp"); + extractor_config.geometry_output_path.append(".osrm.geometry"); + extractor_config.node_output_path.append(".osrm.nodes"); + extractor_config.edge_output_path.append(".osrm.edges"); + extractor_config.edge_graph_output_path.append(".osrm.ebg"); + extractor_config.rtree_nodes_output_path.append(".osrm.ramIndex"); + extractor_config.rtree_leafs_output_path.append(".osrm.fileIndex"); } else { @@ -166,6 +184,12 @@ void ExtractorOptions::GenerateOutputFilesNames(ExtractorConfig &extractor_confi extractor_config.restriction_file_name.replace(pos, 5, ".osrm.restrictions"); extractor_config.names_file_name.replace(pos, 5, ".osrm.names"); extractor_config.timestamp_file_name.replace(pos, 5, ".osrm.timestamp"); + extractor_config.geometry_output_path.replace(pos, 5, ".osrm.geometry"); + extractor_config.node_output_path.replace(pos, 5, ".osrm.nodes"); + extractor_config.edge_output_path.replace(pos, 5, ".osrm.edges"); + extractor_config.edge_graph_output_path.replace(pos, 5, ".osrm.ebg"); + extractor_config.rtree_nodes_output_path.replace(pos, 5, ".osrm.ramIndex"); + extractor_config.rtree_leafs_output_path.replace(pos, 5, ".osrm.fileIndex"); } } else @@ -174,5 +198,11 @@ void ExtractorOptions::GenerateOutputFilesNames(ExtractorConfig &extractor_confi extractor_config.restriction_file_name.replace(pos, 8, ".osrm.restrictions"); extractor_config.names_file_name.replace(pos, 8, ".osrm.names"); extractor_config.timestamp_file_name.replace(pos, 8, ".osrm.timestamp"); + extractor_config.geometry_output_path.replace(pos, 8, ".osrm.geometry"); + extractor_config.node_output_path.replace(pos, 8, ".osrm.nodes"); + extractor_config.edge_output_path.replace(pos, 8, ".osrm.edges"); + extractor_config.edge_graph_output_path.replace(pos, 8, ".osrm.ebg"); + extractor_config.rtree_nodes_output_path.replace(pos, 8, ".osrm.ramIndex"); + extractor_config.rtree_leafs_output_path.replace(pos, 8, ".osrm.fileIndex"); } } diff --git a/extractor/extractor_options.hpp b/extractor/extractor_options.hpp index c164f6a12..6832af545 100644 --- a/extractor/extractor_options.hpp +++ b/extractor/extractor_options.hpp @@ -50,6 +50,12 @@ struct ExtractorConfig std::string restriction_file_name; std::string names_file_name; std::string timestamp_file_name; + std::string geometry_output_path; + std::string edge_output_path; + std::string edge_graph_output_path; + std::string node_output_path; + std::string rtree_nodes_output_path; + std::string rtree_leafs_output_path; unsigned requested_num_threads; }; diff --git a/contractor/speed_profile.hpp b/extractor/speed_profile.hpp similarity index 100% rename from contractor/speed_profile.hpp rename to extractor/speed_profile.hpp diff --git a/features/options/prepare/help.feature b/features/options/prepare/help.feature index 59265e968..896892cda 100644 --- a/features/options/prepare/help.feature +++ b/features/options/prepare/help.feature @@ -13,11 +13,10 @@ Feature: osrm-prepare command line options: help And stdout should contain "--help" And stdout should contain "--config" And stdout should contain "Configuration:" - And stdout should contain "--restrictions" And stdout should contain "--profile" And stdout should contain "--threads" And stdout should contain "--core" - And stdout should contain 17 lines + And stdout should contain 15 lines And it should exit with code 1 Scenario: osrm-prepare - Help, short @@ -29,11 +28,10 @@ Feature: osrm-prepare command line options: help And stdout should contain "--help" And stdout should contain "--config" And stdout should contain "Configuration:" - And stdout should contain "--restrictions" And stdout should contain "--profile" And stdout should contain "--threads" And stdout should contain "--core" - And stdout should contain 17 lines + And stdout should contain 15 lines And it should exit with code 0 Scenario: osrm-prepare - Help, long @@ -45,9 +43,8 @@ Feature: osrm-prepare command line options: help And stdout should contain "--help" And stdout should contain "--config" And stdout should contain "Configuration:" - And stdout should contain "--restrictions" And stdout should contain "--profile" And stdout should contain "--threads" And stdout should contain "--core" - And stdout should contain 17 lines + And stdout should contain 15 lines And it should exit with code 0 diff --git a/features/support/data.rb b/features/support/data.rb index 0c70bb6a5..020a8c6c4 100644 --- a/features/support/data.rb +++ b/features/support/data.rb @@ -278,7 +278,8 @@ def extract_data raise ExtractError.new $?.exitstatus, "osrm-extract exited with code #{$?.exitstatus}." end begin - ["osrm","osrm.names","osrm.restrictions"].each do |file| + ["osrm","osrm.names","osrm.restrictions","osrm.ebg","osrm.edges","osrm.fileIndex","osrm.geometry","osrm.nodes","osrm.ramIndex"].each do |file| + log "Renaming #{osm_file}.#{file} to #{extracted_file}.#{file}", :preprocess File.rename "#{osm_file}.#{file}", "#{extracted_file}.#{file}" end rescue Exception => e @@ -296,14 +297,16 @@ def prepare_data raise PrepareError.new $?.exitstatus, "osrm-prepare exited with code #{$?.exitstatus}." end begin - ["osrm.hsgr","osrm.fileIndex","osrm.geometry","osrm.nodes","osrm.ramIndex","osrm.core"].each do |file| + ["osrm.hsgr","osrm.fileIndex","osrm.geometry","osrm.nodes","osrm.ramIndex","osrm.core","osrm.edges"].each do |file| + log "Renaming #{extracted_file}.#{file} to #{prepared_file}.#{file}", :preprocess File.rename "#{extracted_file}.#{file}", "#{prepared_file}.#{file}" end rescue Exception => e raise FileError.new nil, "failed to rename data file after preparing." end begin - ["osrm.names","osrm.edges","osrm.restrictions"].each do |file| + ["osrm.names","osrm.restrictions","osrm"].each do |file| + log "Copying #{extracted_file}.#{file} to #{prepared_file}.#{file}", :preprocess FileUtils.cp "#{extracted_file}.#{file}", "#{prepared_file}.#{file}" end rescue Exception => e diff --git a/prepare.cpp b/prepare.cpp index 5de320463..caf89dc56 100644 --- a/prepare.cpp +++ b/prepare.cpp @@ -87,7 +87,6 @@ int main(int argc, char *argv[]) } SimpleLogger().Write() << "Input file: " << contractor_config.osrm_input_path.filename().string(); - SimpleLogger().Write() << "Restrictions file: " << contractor_config.restrictions_path.filename().string(); SimpleLogger().Write() << "Profile: " << contractor_config.profile_path.filename().string(); SimpleLogger().Write() << "Threads: " << contractor_config.requested_num_threads; diff --git a/test/.stxxl b/test/.stxxl index b1765e2c0..61cbf0377 100644 --- a/test/.stxxl +++ b/test/.stxxl @@ -1 +1 @@ -disk=/tmp/stxxl,10,syscall +disk=###.stxxl,20,memory diff --git a/unit_tests/algorithms/graph_compressor.cpp b/unit_tests/algorithms/graph_compressor.cpp index 8f500eba5..f9b987cce 100644 --- a/unit_tests/algorithms/graph_compressor.cpp +++ b/unit_tests/algorithms/graph_compressor.cpp @@ -2,7 +2,7 @@ #include "../../data_structures/compressed_edge_container.hpp" #include "../../data_structures/restriction_map.hpp" #include "../../data_structures/node_based_graph.hpp" -#include "../../contractor/speed_profile.hpp" +#include "../../extractor/speed_profile.hpp" #include "../../typedefs.h" #include diff --git a/util/fingerprint.hpp b/util/fingerprint.hpp index 3894fc80f..99576f959 100644 --- a/util/fingerprint.hpp +++ b/util/fingerprint.hpp @@ -52,7 +52,6 @@ class FingerPrint // initialize to {6ba7b810-9dad-11d1-80b4-00c04fd430c8} boost::uuids::uuid named_uuid; - bool has_64_bits; }; diff --git a/util/fingerprint_impl.hpp.in b/util/fingerprint_impl.hpp.in index d6b67a997..390f8ac2c 100644 --- a/util/fingerprint_impl.hpp.in +++ b/util/fingerprint_impl.hpp.in @@ -35,7 +35,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#cmakedefine01 HAS64BITS #cmakedefine MD5PREPARE "${MD5PREPARE}" #cmakedefine MD5RTREE "${MD5RTREE}" #cmakedefine MD5GRAPH "${MD5GRAPH}" @@ -68,7 +67,6 @@ FingerPrint FingerPrint::GetValid() temp_string += fingerprint.md5_objects; fingerprint.named_uuid = gen(temp_string); - fingerprint.has_64_bits = HAS64BITS; return fingerprint; }