Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Merge pull request #1501 from Project-OSRM/refactor/contractor

Browse Source 
First part of the contractor refactor
This commit is contained in:
Patrick Niklaus 2015-05-29 01:08:55 +02:00
commit 0190b5e5af
20 changed files with 724 additions and 949 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CMakeLists.txt
View File

@ -90,7 +90,7 @@ set(
add_library(COORDINATE OBJECT ${CoordinateGlob})
add_library(GITDESCRIPTION OBJECT util/git_sha.cpp)
add_library(OSRM ${OSRMSources} $<TARGET_OBJECTS:ANGLE> $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:GITDESCRIPTION> $<TARGET_OBJECTS:FINGERPRINT> $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:LOGGER> $<TARGET_OBJECTS:PHANTOMNODE> $<TARGET_OBJECTS:EXCEPTION> $<TARGET_OBJECTS:MERCATOR>)
add_library(OSRM ${OSRMSources} $<TARGET_OBJECTS:ANGLE> $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:GITDESCRIPTION> $<TARGET_OBJECTS:FINGERPRINT> $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:LOGGER> $<TARGET_OBJECTS:PHANTOMNODE> $<TARGET_OBJECTS:EXCEPTION> $<TARGET_OBJECTS:MERCATOR> $<TARGET_OBJECTS:IMPORT>)
add_library(FINGERPRINT OBJECT util/fingerprint.cpp)
add_dependencies(FINGERPRINT FingerPrintConfigure)
@ -328,12 +328,10 @@ if(WITH_TOOLS OR BUILD_TOOLS)
if(UNIX AND NOT APPLE)
target_link_libraries(osrm-unlock-all rt)
endif()
add_executable(osrm-check-hsgr tools/check-hsgr.cpp $<TARGET_OBJECTS:FINGERPRINT> $<TARGET_OBJECTS:EXCEPTION> $<TARGET_OBJECTS:LOGGER>)
add_executable(osrm-check-hsgr tools/check-hsgr.cpp $<TARGET_OBJECTS:FINGERPRINT> $<TARGET_OBJECTS:EXCEPTION> $<TARGET_OBJECTS:LOGGER> $<TARGET_OBJECTS:IMPORT>)
target_link_libraries(osrm-check-hsgr ${Boost_LIBRARIES})
add_executable(osrm-springclean tools/springclean.cpp $<TARGET_OBJECTS:FINGERPRINT> $<TARGET_OBJECTS:LOGGER> $<TARGET_OBJECTS:GITDESCRIPTION> $<TARGET_OBJECTS:EXCEPTION>)
target_link_libraries(osrm-springclean ${Boost_LIBRARIES})
add_executable(osrm-graph-compare tools/graph_compare.cpp $<TARGET_OBJECTS:FINGERPRINT> $<TARGET_OBJECTS:IMPORT> $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:LOGGER> $<TARGET_OBJECTS:RESTRICTION> $<TARGET_OBJECTS:EXCEPTION> $<TARGET_OBJECTS:MERCATOR>)
target_link_libraries(osrm-graph-compare ${Boost_LIBRARIES} ${TBB_LIBRARIES})
install(TARGETS osrm-cli DESTINATION bin)
install(TARGETS osrm-io-benchmark DESTINATION bin)

2
contractor/contractor.hpp
View File

@ -186,6 +186,7 @@ class Contractor
}
// clear input vector
input_edge_list.clear();
// FIXME not sure if we need this
edges.shrink_to_fit();
tbb::parallel_sort(edges.begin(), edges.end());
@ -953,7 +954,6 @@ class Contractor
}
std::shared_ptr<ContractorGraph> contractor_graph;
std::vector<ContractorGraph::InputEdge> contracted_edge_list;
stxxl::vector<QueryEdge> external_edge_list;
std::vector<NodeID> orig_node_id_to_new_id_map;
XORFastHash fast_hash;

136
contractor/contractor_options.cpp Normal file
View File

@ -0,0 +1,136 @@
/*
Copyright (c) 2015, Project OSRM contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this list
of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "contractor_options.hpp"
#include "../util/git_sha.hpp"
#include "../util/ini_file.hpp"
#include "../util/simple_logger.hpp"
#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>
#include <tbb/task_scheduler_init.h>
return_code
ContractorOptions::ParseArguments(int argc, char *argv[], ContractorConfig &contractor_config)
{
// declare a group of options that will be allowed only on command line
boost::program_options::options_description generic_options("Options");
generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message")(
"config,c", boost::program_options::value<boost::filesystem::path>(&contractor_config.config_file_path)
->default_value("contractor.ini"),
"Path to a configuration file.");
// declare a group of options that will be allowed both on command line and in config file
boost::program_options::options_description config_options("Configuration");
config_options.add_options()(
"restrictions,r",
boost::program_options::value<boost::filesystem::path>(&contractor_config.restrictions_path),
"Restrictions file in .osrm.restrictions format")(
"profile,p", boost::program_options::value<boost::filesystem::path>(&contractor_config.profile_path)
->default_value("profile.lua"),
"Path to LUA routing profile")(
"threads,t", boost::program_options::value<unsigned int>(&contractor_config.requested_num_threads)
->default_value(tbb::task_scheduler_init::default_num_threads()),
"Number of threads to use");
// hidden options, will be allowed both on command line and in config file, but will not be
// shown to the user
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()(
"input,i", boost::program_options::value<boost::filesystem::path>(&contractor_config.osrm_input_path),
"Input file in .osm, .osm.bz2 or .osm.pbf format");
// positional option
boost::program_options::positional_options_description positional_options;
positional_options.add("input", 1);
// combine above options for parsing
boost::program_options::options_description cmdline_options;
cmdline_options.add(generic_options).add(config_options).add(hidden_options);
boost::program_options::options_description config_file_options;
config_file_options.add(config_options).add(hidden_options);
boost::program_options::options_description visible_options(
"Usage: " + boost::filesystem::basename(argv[0]) + " <input.osrm> [options]");
visible_options.add(generic_options).add(config_options);
// parse command line options
boost::program_options::variables_map option_variables;
boost::program_options::store(boost::program_options::command_line_parser(argc, argv)
.options(cmdline_options)
.positional(positional_options)
.run(),
option_variables);
const auto &temp_config_path = option_variables["config"].as<boost::filesystem::path>();
if (boost::filesystem::is_regular_file(temp_config_path))
{
boost::program_options::store(boost::program_options::parse_config_file<char>(
temp_config_path.string().c_str(), cmdline_options, true),
option_variables);
}
if (option_variables.count("version"))
{
SimpleLogger().Write() << g_GIT_DESCRIPTION;
return return_code::exit;
}
if (option_variables.count("help"))
{
SimpleLogger().Write() << "\n" << visible_options;
return return_code::exit;
}
boost::program_options::notify(option_variables);
if (!option_variables.count("restrictions"))
{
contractor_config.restrictions_path = contractor_config.osrm_input_path.string() + ".restrictions";
}
if (!option_variables.count("input"))
{
SimpleLogger().Write() << "\n" << visible_options;
return return_code::fail;
}
return return_code::ok;
}
void ContractorOptions::GenerateOutputFilesNames(ContractorConfig &contractor_config)
{
contractor_config.node_output_path = contractor_config.osrm_input_path.string() + ".nodes";
contractor_config.edge_output_path = contractor_config.osrm_input_path.string() + ".edges";
contractor_config.geometry_output_path = contractor_config.osrm_input_path.string() + ".geometry";
contractor_config.graph_output_path = contractor_config.osrm_input_path.string() + ".hsgr";
contractor_config.rtree_nodes_output_path = contractor_config.osrm_input_path.string() + ".ramIndex";
contractor_config.rtree_leafs_output_path = contractor_config.osrm_input_path.string() + ".fileIndex";
}

68
contractor/contractor_options.hpp Normal file
View File

@ -0,0 +1,68 @@
/*
Copyright (c) 2015, Project OSRM contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this list
of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef CONTRACTOR_OPTIONS_HPP
#define CONTRACTOR_OPTIONS_HPP
#include <boost/filesystem/path.hpp>
#include <string>
enum class return_code : unsigned
{
ok,
fail,
exit
};
struct ContractorConfig
{
ContractorConfig() noexcept : requested_num_threads(0) {}
boost::filesystem::path config_file_path;
boost::filesystem::path osrm_input_path;
boost::filesystem::path restrictions_path;
boost::filesystem::path profile_path;
std::string node_output_path;
std::string edge_output_path;
std::string geometry_output_path;
std::string graph_output_path;
std::string rtree_nodes_output_path;
std::string rtree_leafs_output_path;
unsigned requested_num_threads;
};
struct ContractorOptions
{
static return_code ParseArguments(int argc, char *argv[], ContractorConfig &extractor_config);
static void GenerateOutputFilesNames(ContractorConfig &extractor_config);
};
#endif // EXTRACTOR_OPTIONS_HPP

55
contractor/edge_based_graph_factory.cpp
View File

@ -40,21 +40,21 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <iomanip>
#include <limits>
EdgeBasedGraphFactory::EdgeBasedGraphFactory(
const std::shared_ptr<NodeBasedDynamicGraph> &node_based_graph,
std::unique_ptr<RestrictionMap> restriction_map,
std::vector<NodeID> &barrier_node_list,
std::vector<NodeID> &traffic_light_node_list,
std::vector<QueryNode> &node_info_list,
SpeedProfileProperties &speed_profile)
EdgeBasedGraphFactory::EdgeBasedGraphFactory(std::shared_ptr<NodeBasedDynamicGraph> node_based_graph,
std::shared_ptr<RestrictionMap> restriction_map,
std::unique_ptr<std::vector<NodeID>> barrier_node_list,
std::unique_ptr<std::vector<NodeID>> traffic_light_node_list,
const std::vector<QueryNode> &node_info_list,
const SpeedProfileProperties &speed_profile)
: speed_profile(speed_profile),
m_number_of_edge_based_nodes(std::numeric_limits<unsigned>::max()),
m_node_info_list(node_info_list), m_node_based_graph(node_based_graph),
m_node_info_list(node_info_list),
m_node_based_graph(std::move(node_based_graph)),
m_restriction_map(std::move(restriction_map)), max_id(0), removed_node_count(0)
{
// insert into unordered sets for fast lookup
m_barrier_nodes.insert(barrier_node_list.begin(), barrier_node_list.end());
m_traffic_lights.insert(traffic_light_node_list.begin(), traffic_light_node_list.end());
m_barrier_nodes.insert(barrier_node_list->begin(), barrier_node_list->end());
m_traffic_lights.insert(traffic_light_node_list->begin(), traffic_light_node_list->end());
}
void EdgeBasedGraphFactory::GetEdgeBasedEdges(DeallocatingVector<EdgeBasedEdge> &output_edge_list)
@ -290,12 +290,33 @@ void EdgeBasedGraphFactory::CompressGeometry()
continue;
}
/*
* reverse_e2 forward_e2
* u <---------- v -----------> w
* ----------> <-----------
* forward_e1 reverse_e1
*
* Will be compressed to:
*
* reverse_e1
* u <---------- w
* ---------->
* forward_e1
*
* If the edges are compatible.
*
*/
const bool reverse_edge_order =
!(m_node_based_graph->GetEdgeData(m_node_based_graph->BeginEdges(node_v)).forward);
const EdgeID forward_e2 = m_node_based_graph->BeginEdges(node_v) + reverse_edge_order;
BOOST_ASSERT(SPECIAL_EDGEID != forward_e2);
BOOST_ASSERT(forward_e2 >= m_node_based_graph->BeginEdges(node_v) &&
forward_e2 < m_node_based_graph->EndEdges(node_v));
const EdgeID reverse_e2 = m_node_based_graph->BeginEdges(node_v) + 1 - reverse_edge_order;
BOOST_ASSERT(SPECIAL_EDGEID != reverse_e2);
BOOST_ASSERT(reverse_e2 >= m_node_based_graph->BeginEdges(node_v) &&
reverse_e2 < m_node_based_graph->EndEdges(node_v));
const EdgeData &fwd_edge_data2 = m_node_based_graph->GetEdgeData(forward_e2);
const EdgeData &rev_edge_data2 = m_node_based_graph->GetEdgeData(reverse_e2);
@ -325,6 +346,11 @@ void EdgeBasedGraphFactory::CompressGeometry()
if ( // TODO: rename to IsCompatibleTo
fwd_edge_data1.IsEqualTo(fwd_edge_data2) && rev_edge_data1.IsEqualTo(rev_edge_data2))
{
BOOST_ASSERT(m_node_based_graph->GetEdgeData(forward_e1).nameID ==
m_node_based_graph->GetEdgeData(reverse_e1).nameID);
BOOST_ASSERT(m_node_based_graph->GetEdgeData(forward_e2).nameID ==
m_node_based_graph->GetEdgeData(reverse_e2).nameID);
// Get distances before graph is modified
const int forward_weight1 = m_node_based_graph->GetEdgeData(forward_e1).distance;
const int forward_weight2 = m_node_based_graph->GetEdgeData(forward_e2).distance;
@ -336,7 +362,7 @@ void EdgeBasedGraphFactory::CompressGeometry()
const int reverse_weight2 = m_node_based_graph->GetEdgeData(reverse_e2).distance;
BOOST_ASSERT(0 != reverse_weight1);
BOOST_ASSERT(0 != forward_weight2);
BOOST_ASSERT(0 != reverse_weight2);
const bool has_node_penalty = m_traffic_lights.find(node_v) != m_traffic_lights.end();
@ -362,11 +388,11 @@ void EdgeBasedGraphFactory::CompressGeometry()
// update any involved turn restrictions
m_restriction_map->FixupStartingTurnRestriction(node_u, node_v, node_w);
m_restriction_map->FixupArrivingTurnRestriction(node_u, node_v, node_w,
m_node_based_graph);
*m_node_based_graph);
m_restriction_map->FixupStartingTurnRestriction(node_w, node_v, node_u);
m_restriction_map->FixupArrivingTurnRestriction(node_w, node_v, node_u,
m_node_based_graph);
*m_node_based_graph);
// store compressed geometry in container
m_geometry_compressor.CompressEdge(
@ -378,8 +404,6 @@ void EdgeBasedGraphFactory::CompressGeometry()
reverse_weight2 + (has_node_penalty ? speed_profile.traffic_signal_penalty : 0));
++removed_node_count;
BOOST_ASSERT(m_node_based_graph->GetEdgeData(forward_e1).nameID ==
m_node_based_graph->GetEdgeData(reverse_e1).nameID);
}
}
SimpleLogger().Write() << "removed " << removed_node_count << " nodes";
@ -415,6 +439,7 @@ void EdgeBasedGraphFactory::RenumberEdges()
for (const auto current_edge : m_node_based_graph->GetAdjacentEdgeRange(current_node))
{
EdgeData &edge_data = m_node_based_graph->GetEdgeData(current_edge);
// FIXME when does that happen? why can we skip here?
if (!edge_data.forward)
{
continue;

17
contractor/edge_based_graph_factory.hpp
View File

@ -59,12 +59,12 @@ class EdgeBasedGraphFactory
struct SpeedProfileProperties;
explicit EdgeBasedGraphFactory(const std::shared_ptr<NodeBasedDynamicGraph> &node_based_graph,
std::unique_ptr<RestrictionMap> restricion_map,
std::vector<NodeID> &barrier_node_list,
std::vector<NodeID> &traffic_light_node_list,
std::vector<QueryNode> &node_info_list,
SpeedProfileProperties &speed_profile);
explicit EdgeBasedGraphFactory(std::shared_ptr<NodeBasedDynamicGraph> node_based_graph,
std::shared_ptr<RestrictionMap> restricion_map,
std::unique_ptr<std::vector<NodeID>> barrier_node_list,
std::unique_ptr<std::vector<NodeID>> traffic_light_node_list,
const std::vector<QueryNode> &node_info_list,
const SpeedProfileProperties &speed_profile);
void Run(const std::string &original_edge_data_filename,
const std::string &geometry_filename,
@ -97,15 +97,16 @@ class EdgeBasedGraphFactory
unsigned m_number_of_edge_based_nodes;
std::vector<QueryNode> m_node_info_list;
std::vector<EdgeBasedNode> m_edge_based_node_list;
DeallocatingVector<EdgeBasedEdge> m_edge_based_edge_list;
const std::vector<QueryNode>& m_node_info_list;
std::shared_ptr<NodeBasedDynamicGraph> m_node_based_graph;
std::shared_ptr<RestrictionMap> m_restriction_map;
std::unordered_set<NodeID> m_barrier_nodes;
std::unordered_set<NodeID> m_traffic_lights;
std::unique_ptr<RestrictionMap> m_restriction_map;
GeometryCompressor m_geometry_compressor;

509
contractor/processing_chain.cpp
View File

@ -48,7 +48,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <boost/filesystem/fstream.hpp>
#include <boost/program_options.hpp>
#include <tbb/task_scheduler_init.h>
#include <tbb/parallel_sort.h>
#include <chrono>
@ -57,166 +56,98 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <thread>
#include <vector>
Prepare::Prepare() : requested_num_threads(1) {}
Prepare::~Prepare() {}
int Prepare::Process(int argc, char *argv[])
int Prepare::Run()
{
LogPolicy::GetInstance().Unmute();
TIMER_START(preparing);
TIMER_START(expansion);
if (!ParseArguments(argc, argv))
{
return 0;
}
if (!boost::filesystem::is_regular_file(input_path))
{
SimpleLogger().Write(logWARNING) << "Input file " << input_path.string() << " not found!";
return 1;
}
if (!boost::filesystem::is_regular_file(profile_path))
{
SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string() << " not found!";
return 1;
}
if (1 > requested_num_threads)
{
SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger";
return 1;
}
const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
SimpleLogger().Write() << "Input file: " << input_path.filename().string();
SimpleLogger().Write() << "Restrictions file: " << restrictions_path.filename().string();
SimpleLogger().Write() << "Profile: " << profile_path.filename().string();
SimpleLogger().Write() << "Threads: " << requested_num_threads;
if (recommended_num_threads != requested_num_threads)
{
SimpleLogger().Write(logWARNING) << "The recommended number of threads is "
<< recommended_num_threads
<< "! This setting may have performance side-effects.";
}
tbb::task_scheduler_init init(requested_num_threads);
LogPolicy::GetInstance().Unmute();
FingerPrint fingerprint_orig;
CheckRestrictionsFile(fingerprint_orig);
boost::filesystem::ifstream input_stream(input_path, std::ios::in | std::ios::binary);
node_filename = input_path.string() + ".nodes";
edge_out = input_path.string() + ".edges";
geometry_filename = input_path.string() + ".geometry";
graph_out = input_path.string() + ".hsgr";
rtree_nodes_path = input_path.string() + ".ramIndex";
rtree_leafs_path = input_path.string() + ".fileIndex";
/*** Setup Scripting Environment ***/
// Create a new lua state
lua_State *lua_state = luaL_newstate();
// Connect LuaBind to this lua state
luabind::open(lua_state);
EdgeBasedGraphFactory::SpeedProfileProperties speed_profile;
if (!SetupScriptingEnvironment(lua_state, speed_profile))
{
return 1;
}
#ifdef WIN32
#pragma message("Memory consumption on Windows can be higher due to different bit packing")
#else
static_assert(sizeof(ImportEdge) == 20,
"changing ImportEdge type has influence on memory consumption!");
static_assert(sizeof(EdgeBasedEdge) == 16,
"changing EdgeBasedEdge type has influence on memory consumption!");
#endif
NodeID number_of_node_based_nodes = readBinaryOSRMGraphFromStream(
input_stream, edge_list, barrier_node_list, traffic_light_list,
&internal_to_external_node_map, restriction_list);
input_stream.close();
if (edge_list.empty())
{
SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
return 1;
}
TIMER_START(preparing);
SimpleLogger().Write() << restriction_list.size() << " restrictions, "
<< barrier_node_list.size() << " bollard nodes, "
<< traffic_light_list.size() << " traffic lights";
// Create a new lua state
std::vector<EdgeBasedNode> node_based_edge_list;
unsigned number_of_edge_based_nodes = 0;
SimpleLogger().Write() << "Generating edge-expanded graph representation";
TIMER_START(expansion);
auto node_based_edge_list = osrm::make_unique<std::vector<EdgeBasedNode>>();;
DeallocatingVector<EdgeBasedEdge> edge_based_edge_list;
auto internal_to_external_node_map = osrm::make_unique<std::vector<QueryNode>>();
auto graph_size =
BuildEdgeExpandedGraph(*internal_to_external_node_map,
*node_based_edge_list, edge_based_edge_list);
// init node_based_edge_list, edge_based_edge_list by edgeList
number_of_edge_based_nodes =
BuildEdgeExpandedGraph(lua_state, number_of_node_based_nodes, node_based_edge_list,
edge_based_edge_list, speed_profile);
lua_close(lua_state);
auto number_of_node_based_nodes = graph_size.first;
auto number_of_edge_based_nodes = graph_size.second;
TIMER_STOP(expansion);
BuildRTree(node_based_edge_list);
SimpleLogger().Write() << "building r-tree ...";
TIMER_START(rtree);
RangebasedCRC32 crc32;
if (crc32.using_hardware())
{
SimpleLogger().Write() << "using hardware based CRC32 computation";
}
else
{
SimpleLogger().Write() << "using software based CRC32 computation";
}
BuildRTree(*node_based_edge_list, *internal_to_external_node_map);
const unsigned crc32_value = crc32(node_based_edge_list);
node_based_edge_list.clear();
node_based_edge_list.shrink_to_fit();
SimpleLogger().Write() << "CRC32: " << crc32_value;
TIMER_STOP(rtree);
WriteNodeMapping();
SimpleLogger().Write() << "writing node map ...";
WriteNodeMapping(std::move(internal_to_external_node_map));
/***
* Contracting the edge-expanded graph
*/
SimpleLogger().Write() << "initializing contractor";
auto contractor =
osrm::make_unique<Contractor>(number_of_edge_based_nodes, edge_based_edge_list);
// Contracting the edge-expanded graph
TIMER_START(contraction);
contractor->Run();
auto contracted_edge_list = osrm::make_unique<DeallocatingVector<QueryEdge>>();
ContractGraph(number_of_edge_based_nodes, edge_based_edge_list, *contracted_edge_list);
TIMER_STOP(contraction);
SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec";
DeallocatingVector<QueryEdge> contracted_edge_list;
contractor->GetEdges(contracted_edge_list);
contractor.reset();
std::size_t number_of_used_edges = WriteContractedGraph(number_of_edge_based_nodes,
std::move(node_based_edge_list),
std::move(contracted_edge_list));
/***
* Sorting contracted edges in a way that the static query graph can read some in in-place.
*/
TIMER_STOP(preparing);
tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end());
const unsigned contracted_edge_count = contracted_edge_list.size();
SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds";
SimpleLogger().Write() << "Expansion : " << (number_of_node_based_nodes / TIMER_SEC(expansion))
<< " nodes/sec and "
<< (number_of_edge_based_nodes / TIMER_SEC(expansion)) << " edges/sec";
SimpleLogger().Write() << "Contraction: "
<< (number_of_edge_based_nodes / TIMER_SEC(contraction))
<< " nodes/sec and " << number_of_used_edges / TIMER_SEC(contraction)
<< " edges/sec";
SimpleLogger().Write() << "finished preprocessing";
return 0;
}
std::size_t Prepare::WriteContractedGraph(unsigned number_of_edge_based_nodes,
std::unique_ptr<std::vector<EdgeBasedNode>> node_based_edge_list,
std::unique_ptr<DeallocatingVector<QueryEdge>> contracted_edge_list)
{
const unsigned crc32_value = CalculateEdgeChecksum(std::move(node_based_edge_list));
// Sorting contracted edges in a way that the static query graph can read some in in-place.
tbb::parallel_sort(contracted_edge_list->begin(), contracted_edge_list->end());
const unsigned contracted_edge_count = contracted_edge_list->size();
SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count
<< " edges";
boost::filesystem::ofstream hsgr_output_stream(graph_out, std::ios::binary);
FingerPrint fingerprint_orig;
boost::filesystem::ofstream hsgr_output_stream(config.graph_output_path, std::ios::binary);
hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint));
const unsigned max_used_node_id = 1 + [&contracted_edge_list]
{
unsigned tmp_max = 0;
for (const QueryEdge &edge : contracted_edge_list)
for (const QueryEdge &edge : *contracted_edge_list)
{
BOOST_ASSERT(SPECIAL_NODEID != edge.source);
BOOST_ASSERT(SPECIAL_NODEID != edge.target);
@ -241,7 +172,7 @@ int Prepare::Process(int argc, char *argv[])
for (const auto node : osrm::irange(0u, max_used_node_id))
{
last_edge = edge;
while ((edge < contracted_edge_count) && (contracted_edge_list[edge].source == node))
while ((edge < contracted_edge_count) && ((*contracted_edge_list)[edge].source == node))
{
++edge;
}
@ -270,19 +201,19 @@ int Prepare::Process(int argc, char *argv[])
hsgr_output_stream.write((char *)&node_array[0],
sizeof(StaticGraph<EdgeData>::NodeArrayEntry) * node_array_size);
}
// serialize all edges
// serialize all edges
SimpleLogger().Write() << "Building edge array";
edge = 0;
int number_of_used_edges = 0;
StaticGraph<EdgeData>::EdgeArrayEntry current_edge;
for (const auto edge : osrm::irange<std::size_t>(0, contracted_edge_list.size()))
for (const auto edge : osrm::irange<std::size_t>(0, contracted_edge_list->size()))
{
// no eigen loops
BOOST_ASSERT(contracted_edge_list[edge].source != contracted_edge_list[edge].target);
current_edge.target = contracted_edge_list[edge].target;
current_edge.data = contracted_edge_list[edge].data;
BOOST_ASSERT((*contracted_edge_list)[edge].source != (*contracted_edge_list)[edge].target);
current_edge.target = (*contracted_edge_list)[edge].target;
current_edge.data = (*contracted_edge_list)[edge].data;
// every target needs to be valid
BOOST_ASSERT(current_edge.target < max_used_node_id);
@ -290,12 +221,12 @@ int Prepare::Process(int argc, char *argv[])
if (current_edge.data.distance <= 0)
{
SimpleLogger().Write(logWARNING) << "Edge: " << edge
<< ",source: " << contracted_edge_list[edge].source
<< ", target: " << contracted_edge_list[edge].target
<< ",source: " << (*contracted_edge_list)[edge].source
<< ", target: " << (*contracted_edge_list)[edge].target
<< ", dist: " << current_edge.data.distance;
SimpleLogger().Write(logWARNING) << "Failed at adjacency list of node "
<< contracted_edge_list[edge].source << "/"
<< (*contracted_edge_list)[edge].source << "/"
<< node_array.size() - 1;
return 1;
}
@ -305,170 +236,54 @@ int Prepare::Process(int argc, char *argv[])
++number_of_used_edges;
}
hsgr_output_stream.close();
TIMER_STOP(preparing);
SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds";
SimpleLogger().Write() << "Expansion : " << (number_of_node_based_nodes / TIMER_SEC(expansion))
<< " nodes/sec and "
<< (number_of_edge_based_nodes / TIMER_SEC(expansion)) << " edges/sec";
SimpleLogger().Write() << "Contraction: "
<< (number_of_edge_based_nodes / TIMER_SEC(contraction))
<< " nodes/sec and " << number_of_used_edges / TIMER_SEC(contraction)
<< " edges/sec";
node_array.clear();
SimpleLogger().Write() << "finished preprocessing";
return 0;
return number_of_used_edges;
}
/**
\brief Parses command line arguments
\param argc count of arguments
\param argv array of arguments
\param result [out] value for exit return value
\return true if everything is ok, false if need to terminate execution
*/
bool Prepare::ParseArguments(int argc, char *argv[])
unsigned Prepare::CalculateEdgeChecksum(std::unique_ptr<std::vector<EdgeBasedNode>> node_based_edge_list)
{
// declare a group of options that will be allowed only on command line
boost::program_options::options_description generic_options("Options");
generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message")(
"config,c", boost::program_options::value<boost::filesystem::path>(&config_file_path)
->default_value("contractor.ini"),
"Path to a configuration file.");
// declare a group of options that will be allowed both on command line and in config file
boost::program_options::options_description config_options("Configuration");
config_options.add_options()(
"restrictions,r",
boost::program_options::value<boost::filesystem::path>(&restrictions_path),
"Restrictions file in .osrm.restrictions format")(
"profile,p", boost::program_options::value<boost::filesystem::path>(&profile_path)
->default_value("profile.lua"),
"Path to LUA routing profile")(
"threads,t", boost::program_options::value<unsigned int>(&requested_num_threads)
->default_value(tbb::task_scheduler_init::default_num_threads()),
"Number of threads to use");
// hidden options, will be allowed both on command line and in config file, but will not be
// shown to the user
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()(
"input,i", boost::program_options::value<boost::filesystem::path>(&input_path),
"Input file in .osm, .osm.bz2 or .osm.pbf format");
// positional option
boost::program_options::positional_options_description positional_options;
positional_options.add("input", 1);
// combine above options for parsing
boost::program_options::options_description cmdline_options;
cmdline_options.add(generic_options).add(config_options).add(hidden_options);
boost::program_options::options_description config_file_options;
config_file_options.add(config_options).add(hidden_options);
boost::program_options::options_description visible_options(
"Usage: " + boost::filesystem::basename(argv[0]) + " <input.osrm> [options]");
visible_options.add(generic_options).add(config_options);
// parse command line options
boost::program_options::variables_map option_variables;
boost::program_options::store(boost::program_options::command_line_parser(argc, argv)
.options(cmdline_options)
.positional(positional_options)
.run(),
option_variables);
const auto &temp_config_path = option_variables["config"].as<boost::filesystem::path>();
if (boost::filesystem::is_regular_file(temp_config_path))
RangebasedCRC32 crc32;
if (crc32.using_hardware())
{
boost::program_options::store(boost::program_options::parse_config_file<char>(
temp_config_path.string().c_str(), cmdline_options, true),
option_variables);
SimpleLogger().Write() << "using hardware based CRC32 computation";
}
else
{
SimpleLogger().Write() << "using software based CRC32 computation";
}
if (option_variables.count("version"))
{
SimpleLogger().Write() << g_GIT_DESCRIPTION;
return false;
}
const unsigned crc32_value = crc32(*node_based_edge_list);
SimpleLogger().Write() << "CRC32: " << crc32_value;
if (option_variables.count("help"))
{
SimpleLogger().Write() << "\n" << visible_options;
return false;
}
boost::program_options::notify(option_variables);
if (!option_variables.count("restrictions"))
{
restrictions_path = std::string(input_path.string() + ".restrictions");
}
if (!option_variables.count("input"))
{
SimpleLogger().Write() << "\n" << visible_options;
return false;
}
return true;
}
/**
\brief Loads and checks file UUIDs
*/
void Prepare::CheckRestrictionsFile(FingerPrint &fingerprint_orig)
{
boost::filesystem::ifstream restriction_stream(restrictions_path, std::ios::binary);
FingerPrint fingerprint_loaded;
unsigned number_of_usable_restrictions = 0;
restriction_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint));
if (!fingerprint_loaded.TestPrepare(fingerprint_orig))
{
SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}
restriction_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned));
restriction_list.resize(number_of_usable_restrictions);
if (number_of_usable_restrictions > 0)
{
restriction_stream.read((char *)&(restriction_list[0]),
number_of_usable_restrictions * sizeof(TurnRestriction));
}
restriction_stream.close();
return crc32_value;
}
/**
\brief Setups scripting environment (lua-scripting)
Also initializes speed profile.
*/
bool Prepare::SetupScriptingEnvironment(
void Prepare::SetupScriptingEnvironment(
lua_State *lua_state, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile)
{
// open utility libraries string library;
luaL_openlibs(lua_state);
// adjust lua load path
luaAddScriptFolderToLoadPath(lua_state, profile_path.string().c_str());
luaAddScriptFolderToLoadPath(lua_state, config.profile_path.string().c_str());
// Now call our function in a lua script
if (0 != luaL_dofile(lua_state, profile_path.string().c_str()))
if (0 != luaL_dofile(lua_state, config.profile_path.string().c_str()))
{
std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl;
return false;
std::stringstream msg;
msg << lua_tostring(lua_state, -1) << " occured in scripting block";
throw osrm::exception(msg.str());
}
if (0 != luaL_dostring(lua_state, "return traffic_signal_penalty\n"))
{
std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl;
return false;
std::stringstream msg;
msg << lua_tostring(lua_state, -1) << " occured in scripting block";
throw osrm::exception(msg.str());
}
speed_profile.traffic_signal_penalty = 10 * lua_tointeger(lua_state, -1);
SimpleLogger().Write(logDEBUG)
@ -476,92 +291,146 @@ bool Prepare::SetupScriptingEnvironment(
if (0 != luaL_dostring(lua_state, "return u_turn_penalty\n"))
{
std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl;
return false;
std::stringstream msg;
msg << lua_tostring(lua_state, -1) << " occured in scripting block";
throw osrm::exception(msg.str());
}
speed_profile.u_turn_penalty = 10 * lua_tointeger(lua_state, -1);
speed_profile.has_turn_penalty_function = lua_function_exists(lua_state, "turn_function");
return true;
}
/**
\brief Build load restrictions from .restriction file
*/
void Prepare::LoadRestrictionMap(const std::unordered_map<NodeID, NodeID> &external_to_internal_node_map,
RestrictionMap &restriction_map)
{
boost::filesystem::ifstream input_stream(config.restrictions_path, std::ios::in | std::ios::binary);
std::vector<TurnRestriction> restriction_list;
loadRestrictionsFromFile(input_stream, external_to_internal_node_map, restriction_list);
SimpleLogger().Write() << " - " << restriction_list.size() << " restrictions.";
restriction_map = RestrictionMap(restriction_list);
}
/**
\brief Build load node based graph from .osrm file and restrictions from .restrictions file
*/
std::shared_ptr<NodeBasedDynamicGraph>
Prepare::LoadNodeBasedGraph(std::vector<NodeID> &barrier_node_list,
std::vector<NodeID> &traffic_light_list,
RestrictionMap &restriction_map,
std::vector<QueryNode>& internal_to_external_node_map)
{
std::vector<ImportEdge> edge_list;
std::unordered_map<NodeID, NodeID> external_to_internal_node_map;
boost::filesystem::ifstream input_stream(config.osrm_input_path, std::ios::in | std::ios::binary);
NodeID number_of_node_based_nodes = loadNodesFromFile(input_stream,
barrier_node_list, traffic_light_list,
internal_to_external_node_map,
external_to_internal_node_map);
SimpleLogger().Write() << " - " << barrier_node_list.size() << " bollard nodes, "
<< traffic_light_list.size() << " traffic lights";
loadEdgesFromFile(input_stream, external_to_internal_node_map, edge_list);
LoadRestrictionMap(external_to_internal_node_map, restriction_map);
if (edge_list.empty())
{
SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
return std::shared_ptr<NodeBasedDynamicGraph>();
}
return NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list);
}
/**
\brief Building an edge-expanded graph from node-based input and turn restrictions
*/
std::size_t
Prepare::BuildEdgeExpandedGraph(lua_State *lua_state,
NodeID number_of_node_based_nodes,
std::pair<std::size_t, std::size_t>
Prepare::BuildEdgeExpandedGraph(std::vector<QueryNode> &internal_to_external_node_map,
std::vector<EdgeBasedNode> &node_based_edge_list,
DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list,
EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile)
DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list)
{
SimpleLogger().Write() << "Generating edge-expanded graph representation";
std::shared_ptr<NodeBasedDynamicGraph> node_based_graph =
NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list);
std::unique_ptr<RestrictionMap> restriction_map =
osrm::make_unique<RestrictionMap>(restriction_list);
std::shared_ptr<EdgeBasedGraphFactory> edge_based_graph_factory =
std::make_shared<EdgeBasedGraphFactory>(node_based_graph, std::move(restriction_map),
barrier_node_list, traffic_light_list,
internal_to_external_node_map, speed_profile);
edge_list.clear();
edge_list.shrink_to_fit();
lua_State *lua_state = luaL_newstate();
luabind::open(lua_state);
edge_based_graph_factory->Run(edge_out, geometry_filename, lua_state);
EdgeBasedGraphFactory::SpeedProfileProperties speed_profile;
restriction_list.clear();
restriction_list.shrink_to_fit();
barrier_node_list.clear();
barrier_node_list.shrink_to_fit();
traffic_light_list.clear();
traffic_light_list.shrink_to_fit();
SetupScriptingEnvironment(lua_state, speed_profile);
auto barrier_node_list = osrm::make_unique<std::vector<NodeID>>();
auto traffic_light_list = osrm::make_unique<std::vector<NodeID>>();
auto restriction_map = std::make_shared<RestrictionMap>();
auto node_based_graph = LoadNodeBasedGraph(*barrier_node_list, *traffic_light_list, *restriction_map, internal_to_external_node_map);
const std::size_t number_of_node_based_nodes = node_based_graph->GetNumberOfNodes();
EdgeBasedGraphFactory edge_based_graph_factory(node_based_graph,
restriction_map,
std::move(barrier_node_list),
std::move(traffic_light_list),
internal_to_external_node_map,
speed_profile);
edge_based_graph_factory.Run(config.edge_output_path, config.geometry_output_path, lua_state);
lua_close(lua_state);
const std::size_t number_of_edge_based_nodes =
edge_based_graph_factory->GetNumberOfEdgeBasedNodes();
edge_based_graph_factory.GetNumberOfEdgeBasedNodes();
BOOST_ASSERT(number_of_edge_based_nodes != std::numeric_limits<unsigned>::max());
#ifndef WIN32
static_assert(sizeof(EdgeBasedEdge) == 16,
"changing ImportEdge type has influence on memory consumption!");
#endif
edge_based_graph_factory->GetEdgeBasedEdges(edge_based_edge_list);
edge_based_graph_factory->GetEdgeBasedNodes(node_based_edge_list);
edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list);
edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list);
edge_based_graph_factory.reset();
node_based_graph.reset();
return std::make_pair(number_of_node_based_nodes, number_of_edge_based_nodes);
}
return number_of_edge_based_nodes;
/**
\brief Build contracted graph.
*/
void Prepare::ContractGraph(const std::size_t number_of_edge_based_nodes,
DeallocatingVector<EdgeBasedEdge>& edge_based_edge_list,
DeallocatingVector<QueryEdge>& contracted_edge_list)
{
Contractor contractor(number_of_edge_based_nodes, edge_based_edge_list);
contractor.Run();
contractor.GetEdges(contracted_edge_list);
}
/**
\brief Writing info on original (node-based) nodes
*/
void Prepare::WriteNodeMapping()
void Prepare::WriteNodeMapping(std::unique_ptr<std::vector<QueryNode>> internal_to_external_node_map)
{
SimpleLogger().Write() << "writing node map ...";
boost::filesystem::ofstream node_stream(node_filename, std::ios::binary);
const unsigned size_of_mapping = internal_to_external_node_map.size();
boost::filesystem::ofstream node_stream(config.node_output_path, std::ios::binary);
const unsigned size_of_mapping = internal_to_external_node_map->size();
node_stream.write((char *)&size_of_mapping, sizeof(unsigned));
if (size_of_mapping > 0)
{
node_stream.write((char *)&(internal_to_external_node_map[0]),
node_stream.write((char *) internal_to_external_node_map->data(),
size_of_mapping * sizeof(QueryNode));
}
node_stream.close();
internal_to_external_node_map.clear();
internal_to_external_node_map.shrink_to_fit();
}
/**
\brief Building rtree-based nearest-neighbor data structure
Saves info to files: '.ramIndex' and '.fileIndex'.
Saves tree into '.ramIndex' and leaves into '.fileIndex'.
*/
void Prepare::BuildRTree(std::vector<EdgeBasedNode> &node_based_edge_list)
void Prepare::BuildRTree(const std::vector<EdgeBasedNode> &node_based_edge_list, const std::vector<QueryNode>& internal_to_external_node_map)
{
SimpleLogger().Write() << "building r-tree ...";
StaticRTree<EdgeBasedNode>(node_based_edge_list, rtree_nodes_path.c_str(),
rtree_leafs_path.c_str(), internal_to_external_node_map);
StaticRTree<EdgeBasedNode>(node_based_edge_list, config.rtree_nodes_output_path.c_str(),
config.rtree_leafs_output_path.c_str(), internal_to_external_node_map);
}

59
contractor/processing_chain.hpp
View File

@ -28,6 +28,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef PROCESSING_CHAIN_HPP
#define PROCESSING_CHAIN_HPP
#include "contractor_options.hpp"
#include "edge_based_graph_factory.hpp"
#include "../data_structures/query_edge.hpp"
#include "../data_structures/static_graph.hpp"
@ -50,46 +51,38 @@ class Prepare
using InputEdge = DynamicGraph<EdgeData>::InputEdge;
using StaticEdge = StaticGraph<EdgeData>::InputEdge;
explicit Prepare();
explicit Prepare(const ContractorConfig& contractor_config)
: config(contractor_config) {}
Prepare(const Prepare &) = delete;
~Prepare();
int Process(int argc, char *argv[]);
int Run();
protected:
bool ParseArguments(int argc, char *argv[]);
void CheckRestrictionsFile(FingerPrint &fingerprint_orig);
bool SetupScriptingEnvironment(lua_State *myLuaState,
void SetupScriptingEnvironment(lua_State *myLuaState,
EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile);
std::size_t BuildEdgeExpandedGraph(lua_State *myLuaState,
NodeID nodeBasedNodeNumber,
std::vector<EdgeBasedNode> &nodeBasedEdgeList,
DeallocatingVector<EdgeBasedEdge> &edgeBasedEdgeList,
EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile);
void WriteNodeMapping();
void BuildRTree(std::vector<EdgeBasedNode> &node_based_edge_list);
void LoadRestrictionMap(const std::unordered_map<NodeID, NodeID> &external_to_internal_node_map,
RestrictionMap &restriction_map);
unsigned CalculateEdgeChecksum(std::unique_ptr<std::vector<EdgeBasedNode>> node_based_edge_list);
void ContractGraph(const std::size_t number_of_edge_based_nodes,
DeallocatingVector<EdgeBasedEdge>& edge_based_edge_list,
DeallocatingVector<QueryEdge>& contracted_edge_list);
std::size_t WriteContractedGraph(unsigned number_of_edge_based_nodes,
std::unique_ptr<std::vector<EdgeBasedNode>> node_based_edge_list,
std::unique_ptr<DeallocatingVector<QueryEdge>> contracted_edge_list);
std::shared_ptr<NodeBasedDynamicGraph> LoadNodeBasedGraph(std::vector<NodeID> &barrier_node_list,
std::vector<NodeID> &traffic_light_list,
RestrictionMap &restriction_map,
std::vector<QueryNode>& internal_to_external_node_map);
std::pair<std::size_t, std::size_t>
BuildEdgeExpandedGraph(std::vector<QueryNode> &internal_to_external_node_map,
std::vector<EdgeBasedNode> &node_based_edge_list,
DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list);
void WriteNodeMapping(std::unique_ptr<std::vector<QueryNode>> internal_to_external_node_map);
void BuildRTree(const std::vector<EdgeBasedNode> &node_based_edge_list,
const std::vector<QueryNode> &internal_to_external_node_map);
private:
std::vector<QueryNode> internal_to_external_node_map;
std::vector<TurnRestriction> restriction_list;
std::vector<NodeID> barrier_node_list;
std::vector<NodeID> traffic_light_list;
std::vector<ImportEdge> edge_list;
unsigned requested_num_threads;
boost::filesystem::path config_file_path;
boost::filesystem::path input_path;
boost::filesystem::path restrictions_path;
boost::filesystem::path preinfo_path;
boost::filesystem::path profile_path;
std::string node_filename;
std::string edge_out;
std::string info_out;
std::string geometry_filename;
std::string graph_out;
std::string rtree_nodes_path;
std::string rtree_leafs_path;
ContractorConfig config;
};
#endif // PROCESSING_CHAIN_HPP

3
data_structures/dynamic_graph.hpp
View File

@ -85,6 +85,9 @@ template <typename EdgeDataT> class DynamicGraph
edge_list.resize(number_of_nodes);
}
/**
* Constructs a DynamicGraph from a list of edges sorted by source node id.
*/
template <class ContainerT> DynamicGraph(const NodeIterator nodes, const ContainerT &graph)
{
number_of_nodes = nodes;

20
data_structures/node_based_graph.hpp
View File

@ -125,8 +125,15 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector<ImportEdge
}
}
// remove duplicate edges
// sort edges by source node id
tbb::parallel_sort(edges_list.begin(), edges_list.end());
// this code removes multi-edges
// my merging mutli-edges bi-directional edges can become directional again!
// Consider the following example:
// a --5-- b
// `--1--^
// After merging we need to split {a, b, 5} into (a, b, 1) and (b, a, 5)
NodeID edge_count = 0;
for (NodeID i = 0; i < edges_list.size();)
{
@ -145,7 +152,7 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector<ImportEdge
forward_edge.data.backward = reverse_edge.data.forward = false;
forward_edge.data.shortcut = reverse_edge.data.shortcut = false;
forward_edge.data.distance = reverse_edge.data.distance = std::numeric_limits<int>::max();
// remove parallel edges
// remove parallel edges and set current distance values
while (i < edges_list.size() && edges_list[i].source == source &&
edges_list[i].target == target)
{
@ -167,17 +174,23 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector<ImportEdge
if (static_cast<int>(forward_edge.data.distance) != std::numeric_limits<int>::max())
{
forward_edge.data.backward = true;
BOOST_ASSERT(edge_count < i);
edges_list[edge_count++] = forward_edge;
}
}
else
{ // insert seperate edges
// this case can only happen if we merged a bi-directional edge with a directional
// edge above, this incrementing i and making it safe to overwrite the next element
// as well
if (static_cast<int>(forward_edge.data.distance) != std::numeric_limits<int>::max())
{
BOOST_ASSERT(edge_count < i);
edges_list[edge_count++] = forward_edge;
}
if (static_cast<int>(reverse_edge.data.distance) != std::numeric_limits<int>::max())
{
BOOST_ASSERT(edge_count < i);
edges_list[edge_count++] = reverse_edge;
}
}
@ -186,9 +199,8 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector<ImportEdge
SimpleLogger().Write() << "merged " << edges_list.size() - edge_count << " edges out of "
<< edges_list.size();
auto graph = std::make_shared<NodeBasedDynamicGraph>(
return std::make_shared<NodeBasedDynamicGraph>(
static_cast<NodeBasedDynamicGraph::NodeIterator>(number_of_nodes), edges_list);
return graph;
}
#endif // NODE_BASED_GRAPH_HPP

7
data_structures/restriction_map.hpp
View File

@ -91,6 +91,7 @@ template <> struct hash<RestrictionTarget>
class RestrictionMap
{
public:
RestrictionMap() = default;
RestrictionMap(const std::vector<TurnRestriction> &restriction_list);
// Replace end v with w in each turn restriction containing u as via node
@ -98,7 +99,7 @@ class RestrictionMap
void FixupArrivingTurnRestriction(const NodeID node_u,
const NodeID node_v,
const NodeID node_w,
const std::shared_ptr<GraphT> &graph)
const GraphT &graph)
{
BOOST_ASSERT(node_u != SPECIAL_NODEID);
BOOST_ASSERT(node_v != SPECIAL_NODEID);
@ -112,9 +113,9 @@ class RestrictionMap
// find all potential start edges. It is more efficent to get a (small) list
// of potential start edges than iterating over all buckets
std::vector<NodeID> predecessors;
for (const EdgeID current_edge_id : graph->GetAdjacentEdgeRange(node_u))
for (const EdgeID current_edge_id : graph.GetAdjacentEdgeRange(node_u))
{
const NodeID target = graph->GetTarget(current_edge_id);
const NodeID target = graph.GetTarget(current_edge_id);
if (node_v != target)
{
predecessors.push_back(target);

2
data_structures/static_rtree.hpp
View File

@ -344,7 +344,7 @@ class StaticRTree
StaticRTree(const StaticRTree &) = delete;
// Construct a packed Hilbert-R-Tree with Kamel-Faloutsos algorithm [1]
explicit StaticRTree(std::vector<EdgeDataT> &input_data_vector,
explicit StaticRTree(const std::vector<EdgeDataT> &input_data_vector,
const std::string tree_node_filename,
const std::string leaf_node_filename,
const std::vector<QueryNode> &coordinate_list)

2
extract.cpp
View File

@ -73,7 +73,7 @@ int main(int argc, char *argv[])
<< " not found!";
return 1;
}
return extractor().run(extractor_config);
return extractor(extractor_config).run();
}
catch (const std::exception &e)
{

20
extractor/extractor.cpp
View File

@ -82,7 +82,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
* .restrictions : Turn restrictions that are used my osrm-prepare to construct the edge-expanded graph
*
*/
int extractor::run(const ExtractorConfig &extractor_config)
int extractor::run()
{
try
{
@ -91,20 +91,20 @@ int extractor::run(const ExtractorConfig &extractor_config)
const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
const auto number_of_threads =
std::min(recommended_num_threads, extractor_config.requested_num_threads);
std::min(recommended_num_threads, config.requested_num_threads);
tbb::task_scheduler_init init(number_of_threads);
SimpleLogger().Write() << "Input file: " << extractor_config.input_path.filename().string();
SimpleLogger().Write() << "Profile: " << extractor_config.profile_path.filename().string();
SimpleLogger().Write() << "Input file: " << config.input_path.filename().string();
SimpleLogger().Write() << "Profile: " << config.profile_path.filename().string();
SimpleLogger().Write() << "Threads: " << number_of_threads;
// setup scripting environment
ScriptingEnvironment scripting_environment(extractor_config.profile_path.string().c_str());
ScriptingEnvironment scripting_environment(config.profile_path.string().c_str());
ExtractionContainers extraction_containers;
auto extractor_callbacks = osrm::make_unique<ExtractorCallbacks>(extraction_containers);
const osmium::io::File input_file(extractor_config.input_path.string());
const osmium::io::File input_file(config.input_path.string());
osmium::io::Reader reader(input_file);
const osmium::io::Header header = reader.header();
@ -131,7 +131,7 @@ int extractor::run(const ExtractorConfig &extractor_config)
}
SimpleLogger().Write() << "timestamp: " << timestamp;
boost::filesystem::ofstream timestamp_out(extractor_config.timestamp_file_name);
boost::filesystem::ofstream timestamp_out(config.timestamp_file_name);
timestamp_out.write(timestamp.c_str(), timestamp.length());
timestamp_out.close();
@ -236,12 +236,12 @@ int extractor::run(const ExtractorConfig &extractor_config)
return 1;
}
extraction_containers.PrepareData(extractor_config.output_file_name,
extractor_config.restriction_file_name);
extraction_containers.PrepareData(config.output_file_name,
config.restriction_file_name);
TIMER_STOP(extracting);
SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s";
SimpleLogger().Write() << "To prepare the data for routing, run: "
<< "./osrm-prepare " << extractor_config.output_file_name
<< "./osrm-prepare " << config.output_file_name
<< std::endl;
}
catch (std::exception &e)

9
extractor/extractor.hpp
View File

@ -30,8 +30,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "extractor_options.hpp"
struct extractor
class extractor
{
int run(const ExtractorConfig &extractor_config);
public:
extractor(const ExtractorConfig &extractor_config)
: config(extractor_config) {}
int run();
private:
ExtractorConfig config;
};
#endif /* EXTRACTOR_HPP */

2
features/options/prepare/help.feature
View File

@ -17,7 +17,7 @@ Feature: osrm-prepare command line options: help
And stdout should contain "--profile"
And stdout should contain "--threads"
And stdout should contain 15 lines
And it should exit with code 0
And it should exit with code 1
Scenario: osrm-prepare - Help, short
When I run "osrm-prepare -h"

60
prepare.cpp
View File

@ -26,10 +26,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "contractor/processing_chain.hpp"
#include "contractor/contractor_options.hpp"
#include "util/simple_logger.hpp"
#include <boost/program_options/errors.hpp>
#include <tbb/task_scheduler_init.h>
#include <exception>
#include <ostream>
@ -37,21 +40,64 @@ int main(int argc, char *argv[])
{
try
{
return Prepare().Process(argc, argv);
}
catch (boost::program_options::too_many_positional_options_error &)
LogPolicy::GetInstance().Unmute();
ContractorConfig contractor_config;
const return_code result = ContractorOptions::ParseArguments(argc, argv, contractor_config);
if (return_code::fail == result)
{
SimpleLogger().Write(logWARNING) << "Only one file can be specified";
return 1;
}
catch (boost::program_options::error &e)
if (return_code::exit == result)
{
SimpleLogger().Write(logWARNING) << e.what();
return 0;
}
ContractorOptions::GenerateOutputFilesNames(contractor_config);
if (1 > contractor_config.requested_num_threads)
{
SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger";
return 1;
}
const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
if (recommended_num_threads != contractor_config.requested_num_threads)
{
SimpleLogger().Write(logWARNING) << "The recommended number of threads is "
<< recommended_num_threads
<< "! This setting may have performance side-effects.";
}
if (!boost::filesystem::is_regular_file(contractor_config.osrm_input_path))
{
SimpleLogger().Write(logWARNING)
<< "Input file " << contractor_config.osrm_input_path.string() << " not found!";
return 1;
}
if (!boost::filesystem::is_regular_file(contractor_config.profile_path))
{
SimpleLogger().Write(logWARNING) << "Profile " << contractor_config.profile_path.string()
<< " not found!";
return 1;
}
SimpleLogger().Write() << "Input file: " << contractor_config.osrm_input_path.filename().string();
SimpleLogger().Write() << "Restrictions file: " << contractor_config.restrictions_path.filename().string();
SimpleLogger().Write() << "Profile: " << contractor_config.profile_path.filename().string();
SimpleLogger().Write() << "Threads: " << contractor_config.requested_num_threads;
tbb::task_scheduler_init init(contractor_config.requested_num_threads);
return Prepare(contractor_config).Run();
}
catch (const std::exception &e)
{
SimpleLogger().Write(logWARNING) << "Exception occured: " << e.what() << std::endl;
SimpleLogger().Write(logWARNING) << "[exception] " << e.what();
return 1;
}
}

117
tools/components.cpp
View File

@ -76,55 +76,25 @@ void DeleteFileIfExists(const std::string &file_name)
}
}
int main(int argc, char *argv[])
void LoadRestrictions(const char* path,
std::unique_ptr<std::unordered_map<NodeID, NodeID>> ext_to_int_id_map,
std::vector<TurnRestriction>& restriction_list)
{
std::vector<QueryNode> coordinate_list;
std::vector<TurnRestriction> restriction_list;
std::vector<NodeID> bollard_node_list;
std::vector<NodeID> traffic_lights_list;
LogPolicy::GetInstance().Unmute();
try
std::ifstream input_stream(path, std::ios::binary);
if (!input_stream.is_open())
{
// enable logging
if (argc < 3)
{
SimpleLogger().Write(logWARNING) << "usage:\n" << argv[0]
<< " <osrm> <osrm.restrictions>";
return -1;
throw osrm::exception("Cannot open restriction file");
}
loadRestrictionsFromFile(input_stream, *ext_to_int_id_map, restriction_list);
}
SimpleLogger().Write() << "Using restrictions from file: " << argv[2];
std::ifstream restriction_ifstream(argv[2], std::ios::binary);
const FingerPrint fingerprint_orig;
FingerPrint fingerprint_loaded;
restriction_ifstream.read(reinterpret_cast<char *>(&fingerprint_loaded),
sizeof(FingerPrint));
// check fingerprint and warn if necessary
if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig))
{
SimpleLogger().Write(logWARNING) << argv[2] << " was prepared with a different build. "
"Reprocess to get rid of this warning.";
}
if (!restriction_ifstream.good())
{
throw osrm::exception("Could not access <osrm-restrictions> files");
}
uint32_t usable_restrictions = 0;
restriction_ifstream.read(reinterpret_cast<char *>(&usable_restrictions), sizeof(uint32_t));
restriction_list.resize(usable_restrictions);
// load restrictions
if (usable_restrictions > 0)
{
restriction_ifstream.read(reinterpret_cast<char *>(&restriction_list[0]),
usable_restrictions * sizeof(TurnRestriction));
}
restriction_ifstream.close();
std::ifstream input_stream(argv[1], std::ifstream::in | std::ifstream::binary);
std::size_t LoadGraph(const char* path,
std::vector<QueryNode>& coordinate_list,
std::vector<NodeID>& barrier_node_list,
std::unordered_map<NodeID, NodeID>& ext_to_int_id_map,
std::vector<TarjanEdge>& graph_edge_list)
{
std::ifstream input_stream(path, std::ifstream::in | std::ifstream::binary);
if (!input_stream.is_open())
{
throw osrm::exception("Cannot open osrm file");
@ -132,24 +102,19 @@ int main(int argc, char *argv[])
// load graph data
std::vector<ImportEdge> edge_list;
const NodeID number_of_nodes =
readBinaryOSRMGraphFromStream(input_stream, edge_list, bollard_node_list,
traffic_lights_list, &coordinate_list, restriction_list);
input_stream.close();
std::vector<NodeID> traffic_light_node_list;
BOOST_ASSERT_MSG(restriction_list.size() == usable_restrictions,
"size of restriction_list changed");
auto number_of_nodes = loadNodesFromFile(input_stream, barrier_node_list,
traffic_light_node_list,
coordinate_list,
ext_to_int_id_map);
SimpleLogger().Write() << restriction_list.size() << " restrictions, "
<< bollard_node_list.size() << " bollard nodes, "
<< traffic_lights_list.size() << " traffic lights";
auto number_of_edges = loadEdgesFromFile(input_stream, ext_to_int_id_map, edge_list);
traffic_lights_list.clear();
traffic_lights_list.shrink_to_fit();
traffic_light_node_list.clear();
traffic_light_node_list.shrink_to_fit();
// Building an node-based graph
std::vector<TarjanEdge> graph_edge_list;
// DeallocatingVector<TarjanEdge> graph_edge_list;
for (const auto &input_edge : edge_list)
{
if (input_edge.source == input_edge.target)
@ -168,10 +133,33 @@ int main(int argc, char *argv[])
(std::max)(input_edge.weight, 1), input_edge.name_id);
}
}
edge_list.clear();
edge_list.shrink_to_fit();
BOOST_ASSERT_MSG(0 == edge_list.size() && 0 == edge_list.capacity(),
"input edge vector not properly deallocated");
return number_of_nodes;
}
int main(int argc, char *argv[])
{
std::vector<QueryNode> coordinate_list;
std::vector<TurnRestriction> restriction_list;
std::vector<NodeID> barrier_node_list;
auto ext_to_int_id_map = osrm::make_unique<std::unordered_map<NodeID, NodeID>>();
LogPolicy::GetInstance().Unmute();
try
{
// enable logging
if (argc < 3)
{
SimpleLogger().Write(logWARNING) << "usage:\n" << argv[0]
<< " <osrm> <osrm.restrictions>";
return -1;
}
SimpleLogger().Write() << "Using restrictions from file: " << argv[2];
std::vector<TarjanEdge> graph_edge_list;
auto number_of_nodes = LoadGraph(argv[1], coordinate_list, barrier_node_list, *ext_to_int_id_map, graph_edge_list);
LoadRestrictions(argv[2], std::move(ext_to_int_id_map), restriction_list);
tbb::parallel_sort(graph_edge_list.begin(), graph_edge_list.end());
const auto graph = std::make_shared<TarjanGraph>(number_of_nodes, graph_edge_list);
@ -181,9 +169,8 @@ int main(int argc, char *argv[])
SimpleLogger().Write() << "Starting SCC graph traversal";
RestrictionMap restriction_map(restriction_list);
auto tarjan = osrm::make_unique<TarjanSCC<TarjanGraph>>(graph,
restriction_map,
bollard_node_list);
auto tarjan = osrm::make_unique<TarjanSCC<TarjanGraph>>(graph, restriction_map,
barrier_node_list);
tarjan->run();
SimpleLogger().Write() << "identified: " << tarjan->get_number_of_components()
<< " many components";

199
tools/graph_compare.cpp
View File

@ -1,199 +0,0 @@
#include "../data_structures/dynamic_graph.hpp"
#include "../data_structures/import_edge.hpp"
#include "../data_structures/query_node.hpp"
#include "../data_structures/restriction.hpp"
#include "../data_structures/static_graph.hpp"
#include "../util/fingerprint.hpp"
#include "../util/graph_loader.hpp"
#include "../util/integer_range.hpp"
#include "../util/make_unique.hpp"
#include "../util/osrm_exception.hpp"
#include "../util/simple_logger.hpp"
#include "../typedefs.h"
#include <algorithm>
#include <fstream>
struct TarjanEdgeData
{
TarjanEdgeData() : distance(INVALID_EDGE_WEIGHT), name_id(INVALID_NAMEID) {}
TarjanEdgeData(unsigned distance, unsigned name_id) : distance(distance), name_id(name_id) {}
unsigned distance;
unsigned name_id;
};
using StaticTestGraph = StaticGraph<TarjanEdgeData>;
using DynamicTestGraph = StaticGraph<TarjanEdgeData>;
using StaticEdge = StaticTestGraph::InputEdge;
using DynamicEdge = DynamicTestGraph::InputEdge;
int main(int argc, char *argv[])
{
std::vector<QueryNode> coordinate_list;
std::vector<TurnRestriction> restriction_list;
std::vector<NodeID> bollard_node_list;
std::vector<NodeID> traffic_lights_list;
LogPolicy::GetInstance().Unmute();
try
{
// enable logging
if (argc < 3)
{
SimpleLogger().Write(logWARNING) << "usage:\n" << argv[0]
<< " <osrm> <osrm.restrictions>";
return -1;
}
SimpleLogger().Write() << "Using restrictions from file: " << argv[2];
std::ifstream restriction_ifstream(argv[2], std::ios::binary);
const FingerPrint fingerprint_orig;
FingerPrint fingerprint_loaded;
restriction_ifstream.read(reinterpret_cast<char *>(&fingerprint_loaded),
sizeof(FingerPrint));
// check fingerprint and warn if necessary
if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig))
{
SimpleLogger().Write(logWARNING) << argv[2] << " was prepared with a different build. "
"Reprocess to get rid of this warning.";
}
if (!restriction_ifstream.good())
{
throw osrm::exception("Could not access <osrm-restrictions> files");
}
uint32_t usable_restrictions = 0;
restriction_ifstream.read(reinterpret_cast<char *>(&usable_restrictions), sizeof(uint32_t));
restriction_list.resize(usable_restrictions);
// load restrictions
if (usable_restrictions > 0)
{
restriction_ifstream.read(reinterpret_cast<char *>(&restriction_list[0]),
usable_restrictions * sizeof(TurnRestriction));
}
restriction_ifstream.close();
std::ifstream input_stream(argv[1], std::ifstream::in | std::ifstream::binary);
if (!input_stream.is_open())
{
throw osrm::exception("Cannot open osrm file");
}
// load graph data
std::vector<ImportEdge> edge_list;
const NodeID number_of_nodes =
readBinaryOSRMGraphFromStream(input_stream, edge_list, bollard_node_list,
traffic_lights_list, &coordinate_list, restriction_list);
input_stream.close();
BOOST_ASSERT_MSG(restriction_list.size() == usable_restrictions,
"size of restriction_list changed");
SimpleLogger().Write() << restriction_list.size() << " restrictions, "
<< bollard_node_list.size() << " bollard nodes, "
<< traffic_lights_list.size() << " traffic lights";
traffic_lights_list.clear();
traffic_lights_list.shrink_to_fit();
// Building an node-based graph
std::vector<StaticEdge> static_graph_edge_list;
std::vector<DynamicEdge> dynamic_graph_edge_list;
for (const auto &input_edge : edge_list)
{
if (input_edge.source == input_edge.target)
{
continue;
}
if (input_edge.forward)
{
static_graph_edge_list.emplace_back(input_edge.source, input_edge.target,
(std::max)(input_edge.weight, 1),
input_edge.name_id);
dynamic_graph_edge_list.emplace_back(input_edge.source, input_edge.target,
(std::max)(input_edge.weight, 1),
input_edge.name_id);
}
if (input_edge.backward)
{
dynamic_graph_edge_list.emplace_back(input_edge.target, input_edge.source,
(std::max)(input_edge.weight, 1),
input_edge.name_id);
static_graph_edge_list.emplace_back(input_edge.target, input_edge.source,
(std::max)(input_edge.weight, 1),
input_edge.name_id);
}
}
edge_list.clear();
edge_list.shrink_to_fit();
BOOST_ASSERT_MSG(0 == edge_list.size() && 0 == edge_list.capacity(),
"input edge vector not properly deallocated");
tbb::parallel_sort(static_graph_edge_list.begin(), static_graph_edge_list.end());
tbb::parallel_sort(dynamic_graph_edge_list.begin(), dynamic_graph_edge_list.end());
auto static_graph =
osrm::make_unique<StaticTestGraph>(number_of_nodes, static_graph_edge_list);
auto dynamic_graph =
osrm::make_unique<DynamicTestGraph>(number_of_nodes, dynamic_graph_edge_list);
SimpleLogger().Write() << "Starting static/dynamic graph comparison";
BOOST_ASSERT(static_graph->GetNumberOfNodes() == dynamic_graph->GetNumberOfNodes());
BOOST_ASSERT(static_graph->GetNumberOfEdges() == dynamic_graph->GetNumberOfEdges());
for (const auto node : osrm::irange(0u, static_graph->GetNumberOfNodes()))
{
const auto static_range = static_graph->GetAdjacentEdgeRange(node);
const auto dynamic_range = dynamic_graph->GetAdjacentEdgeRange(node);
SimpleLogger().Write() << "checking node " << node << "/"
<< static_graph->GetNumberOfNodes();
BOOST_ASSERT(static_range.size() == dynamic_range.size());
const auto static_begin = static_graph->BeginEdges(node);
const auto dynamic_begin = dynamic_graph->BeginEdges(node);
// check raw interface
for (const auto i : osrm::irange(0u, static_range.size()))
{
const auto static_target = static_graph->GetTarget(static_begin + i);
const auto dynamic_target = dynamic_graph->GetTarget(dynamic_begin + i);
BOOST_ASSERT(static_target == dynamic_target);
const auto static_data = static_graph->GetEdgeData(static_begin + i);
const auto dynamic_data = dynamic_graph->GetEdgeData(dynamic_begin + i);
BOOST_ASSERT(static_data.distance == dynamic_data.distance);
BOOST_ASSERT(static_data.name_id == dynamic_data.name_id);
}
// check range interface
std::vector<EdgeID> static_target_ids, dynamic_target_ids;
std::vector<TarjanEdgeData> static_edge_data, dynamic_edge_data;
for (const auto static_id : static_range)
{
static_target_ids.push_back(static_graph->GetTarget(static_id));
static_edge_data.push_back(static_graph->GetEdgeData(static_id));
}
for (const auto dynamic_id : dynamic_range)
{
dynamic_target_ids.push_back(dynamic_graph->GetTarget(dynamic_id));
dynamic_edge_data.push_back(dynamic_graph->GetEdgeData(dynamic_id));
}
BOOST_ASSERT(static_target_ids.size() == dynamic_target_ids.size());
BOOST_ASSERT(std::equal(std::begin(static_target_ids), std::end(static_target_ids),
std::begin(dynamic_target_ids)));
}
SimpleLogger().Write() << "Graph comparison finished successfully";
}
catch (const std::exception &e)
{
SimpleLogger().Write(logWARNING) << "[exception] " << e.what();
}
return 0;
}

340
util/graph_loader.hpp
View File

@ -52,269 +52,34 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <unordered_map>
#include <vector>
template <typename EdgeT>
NodeID read_undirected_osrm_stream(std::istream &input_stream,
std::vector<EdgeT> &edge_list,
std::vector<FixedPointCoordinate> &coordinate_list)
/**
* Reads the .restrictions file and loads it to a vector.
* The since the restrictions reference nodes using their external node id,
* we need to renumber it to the new internal id.
*/
unsigned loadRestrictionsFromFile(std::istream& input_stream,
const std::unordered_map<NodeID, NodeID>& ext_to_int_id_map,
std::vector<TurnRestriction>& restriction_list)
{
const FingerPrint fingerprint_orig;
FingerPrint fingerprint_loaded;
input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(FingerPrint));
if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig))
unsigned number_of_usable_restrictions = 0;
input_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint));
if (!fingerprint_loaded.TestPrepare(fingerprint_orig))
{
SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n"
SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}
std::unordered_map<NodeID, NodeID> ext_to_int_id_map;
NodeID n;
input_stream.read(reinterpret_cast<char *>(&n), sizeof(NodeID));
SimpleLogger().Write() << "Importing n = " << n << " nodes ";
ExternalMemoryNode current_node;
for (NodeID i = 0; i < n; ++i)
input_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned));
restriction_list.resize(number_of_usable_restrictions);
if (number_of_usable_restrictions > 0)
{
input_stream.read(reinterpret_cast<char *>(&current_node), sizeof(ExternalMemoryNode));
coordinate_list.emplace_back(current_node.lat, current_node.lon);
ext_to_int_id_map.emplace(current_node.node_id, i);
// if (current_node.barrier)
// {
// barrier_node_list.emplace_back(i);
// }
// if (current_node.traffic_lights)
// {
// traffic_light_node_list.emplace_back(i);
// }
input_stream.read((char *) restriction_list.data(),
number_of_usable_restrictions * sizeof(TurnRestriction));
}
// tighten vector sizes
// barrier_node_list.shrink_to_fit();
// traffic_light_node_list.shrink_to_fit();
// renumber nodes in turn restrictions
// for (TurnRestriction &current_restriction : restriction_list)
// {
// auto internal_id_iter = ext_to_int_id_map.find(current_restriction.from.node);
// if (internal_id_iter == ext_to_int_id_map.end())
// {
// SimpleLogger().Write(logDEBUG) << "Unmapped from node " <<
// current_restriction.from.node
// << " of restriction";
// continue;
// }
// current_restriction.from.node = internal_id_iter->second;
// internal_id_iter = ext_to_int_id_map.find(current_restriction.via.node);
// if (internal_id_iter == ext_to_int_id_map.end())
// {
// SimpleLogger().Write(logDEBUG) << "Unmapped via node " <<
// current_restriction.via.node
// << " of restriction";
// continue;
// }
// current_restriction.via.node = internal_id_iter->second;
// internal_id_iter = ext_to_int_id_map.find(current_restriction.to.node);
// if (internal_id_iter == ext_to_int_id_map.end())
// {
// SimpleLogger().Write(logDEBUG) << "Unmapped to node " << current_restriction.to.node
// << " of restriction";
// continue;
// }
// current_restriction.to.node = internal_id_iter->second;
// }
EdgeWeight weight;
NodeID source, target;
unsigned nameID;
int length;
short dir; // direction (0 = open, 1 = forward, 2+ = open)
bool is_roundabout, ignore_in_grid, is_access_restricted, is_split;
TravelMode travel_mode;
EdgeID m;
input_stream.read(reinterpret_cast<char *>(&m), sizeof(unsigned));
edge_list.reserve(m);
SimpleLogger().Write() << " and " << m << " edges ";
for (EdgeID i = 0; i < m; ++i)
{
input_stream.read(reinterpret_cast<char *>(&source), sizeof(unsigned));
input_stream.read(reinterpret_cast<char *>(&target), sizeof(unsigned));
input_stream.read(reinterpret_cast<char *>(&length), sizeof(int));
input_stream.read(reinterpret_cast<char *>(&dir), sizeof(short));
input_stream.read(reinterpret_cast<char *>(&weight), sizeof(int));
input_stream.read(reinterpret_cast<char *>(&nameID), sizeof(unsigned));
input_stream.read(reinterpret_cast<char *>(&is_roundabout), sizeof(bool));
input_stream.read(reinterpret_cast<char *>(&ignore_in_grid), sizeof(bool));
input_stream.read(reinterpret_cast<char *>(&is_access_restricted), sizeof(bool));
input_stream.read(reinterpret_cast<char *>(&travel_mode), sizeof(TravelMode));
input_stream.read(reinterpret_cast<char *>(&is_split), sizeof(bool));
BOOST_ASSERT_MSG(length > 0, "loaded null length edge");
BOOST_ASSERT_MSG(weight > 0, "loaded null weight");
BOOST_ASSERT_MSG(0 <= dir && dir <= 2, "loaded bogus direction");
// bool forward = true;
// bool backward = true;
// if (1 == dir)
// {
// backward = false;
// }
// if (2 == dir)
// {
// forward = false;
// }
// translate the external NodeIDs to internal IDs
auto internal_id_iter = ext_to_int_id_map.find(source);
if (ext_to_int_id_map.find(source) == ext_to_int_id_map.end())
{
#ifndef NDEBUG
SimpleLogger().Write(logWARNING) << " unresolved source NodeID: " << source;
#endif
continue;
}
source = internal_id_iter->second;
internal_id_iter = ext_to_int_id_map.find(target);
if (ext_to_int_id_map.find(target) == ext_to_int_id_map.end())
{
#ifndef NDEBUG
SimpleLogger().Write(logWARNING) << "unresolved target NodeID : " << target;
#endif
continue;
}
target = internal_id_iter->second;
BOOST_ASSERT_MSG(source != SPECIAL_NODEID && target != SPECIAL_NODEID,
"nonexisting source or target");
if (source > target)
{
std::swap(source, target);
// std::swap(forward, backward);
}
edge_list.emplace_back(source, target);
}
ext_to_int_id_map.clear();
tbb::parallel_sort(edge_list.begin(), edge_list.end());
// for (unsigned i = 1; i < edge_list.size(); ++i)
// {
// if ((edge_list[i - 1].target == edge_list[i].target) &&
// (edge_list[i - 1].source == edge_list[i].source))
// {
// const bool edge_flags_equivalent = (edge_list[i - 1].forward == edge_list[i].forward)
// &&
// (edge_list[i - 1].backward ==
// edge_list[i].backward);
// const bool edge_flags_are_superset1 =
// (edge_list[i - 1].forward && edge_list[i - 1].backward) &&
// (edge_list[i].forward != edge_list[i].backward);
// const bool edge_flags_are_superset_2 =
// (edge_list[i].forward && edge_list[i].backward) &&
// (edge_list[i - 1].forward != edge_list[i - 1].backward);
// if (edge_flags_equivalent)
// {
// edge_list[i].weight = std::min(edge_list[i - 1].weight, edge_list[i].weight);
// edge_list[i - 1].source = SPECIAL_NODEID;
// }
// else if (edge_flags_are_superset1)
// {
// if (edge_list[i - 1].weight <= edge_list[i].weight)
// {
// // edge i-1 is smaller and goes in both directions. Throw away the other edge
// edge_list[i].source = SPECIAL_NODEID;
// }
// else
// {
// // edge i-1 is open in both directions, but edge i is smaller in one
// direction.
// // Close edge i-1 in this direction
// edge_list[i - 1].forward = !edge_list[i].forward;
// edge_list[i - 1].backward = !edge_list[i].backward;
// }
// }
// else if (edge_flags_are_superset_2)
// {
// if (edge_list[i - 1].weight <= edge_list[i].weight)
// {
// // edge i-1 is smaller for one direction. edge i is open in both. close edge
// i
// // in the other direction
// edge_list[i].forward = !edge_list[i - 1].forward;
// edge_list[i].backward = !edge_list[i - 1].backward;
// }
// else
// {
// // edge i is smaller and goes in both direction. Throw away edge i-1
// edge_list[i - 1].source = SPECIAL_NODEID;
// }
// }
// }
// }
// const auto new_end_iter =
// std::remove_if(edge_list.begin(), edge_list.end(), [](const EdgeT &edge)
// {
// return edge.source == SPECIAL_NODEID || edge.target == SPECIAL_NODEID;
// });
// edge_list.erase(new_end_iter, edge_list.end()); // remove excess candidates.
// edge_list.shrink_to_fit();
SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges";
return n;
}
template <typename EdgeT>
NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream,
std::vector<EdgeT> &edge_list,
std::vector<NodeID> &barrier_node_list,
std::vector<NodeID> &traffic_light_node_list,
std::vector<QueryNode> *int_to_ext_node_id_map,
std::vector<TurnRestriction> &restriction_list)
{
const FingerPrint fingerprint_orig;
FingerPrint fingerprint_loaded;
input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(FingerPrint));
if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig))
{
SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}
std::unordered_map<NodeID, NodeID> ext_to_int_id_map;
NodeID n;
input_stream.read(reinterpret_cast<char *>(&n), sizeof(NodeID));
SimpleLogger().Write() << "Importing n = " << n << " nodes ";
ExternalMemoryNode current_node;
for (NodeID i = 0; i < n; ++i)
{
input_stream.read(reinterpret_cast<char *>(&current_node), sizeof(ExternalMemoryNode));
int_to_ext_node_id_map->emplace_back(current_node.lat, current_node.lon,
current_node.node_id);
ext_to_int_id_map.emplace(current_node.node_id, i);
if (current_node.barrier)
{
barrier_node_list.emplace_back(i);
}
if (current_node.traffic_lights)
{
traffic_light_node_list.emplace_back(i);
}
}
// tighten vector sizes
barrier_node_list.shrink_to_fit();
traffic_light_node_list.shrink_to_fit();
// renumber nodes in turn restrictions
// renumber ids referenced in restrictions
for (TurnRestriction &current_restriction : restriction_list)
{
auto internal_id_iter = ext_to_int_id_map.find(current_restriction.from.node);
@ -346,6 +111,70 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream,
current_restriction.to.node = internal_id_iter->second;
}
return number_of_usable_restrictions;
}
/**
* Reads the beginning of an .osrm file and produces:
* - list of barrier nodes
* - list of traffic lights
* - index to original node id map
* - original node id to index map
*/
NodeID loadNodesFromFile(std::istream &input_stream,
std::vector<NodeID> &barrier_node_list,
std::vector<NodeID> &traffic_light_node_list,
std::vector<QueryNode> &int_to_ext_node_id_map,
std::unordered_map<NodeID, NodeID> &ext_to_int_id_map)
{
const FingerPrint fingerprint_orig;
FingerPrint fingerprint_loaded;
input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(FingerPrint));
if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig))
{
SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}
NodeID n;
input_stream.read(reinterpret_cast<char *>(&n), sizeof(NodeID));
SimpleLogger().Write() << "Importing n = " << n << " nodes ";
ExternalMemoryNode current_node;
for (NodeID i = 0; i < n; ++i)
{
input_stream.read(reinterpret_cast<char *>(&current_node), sizeof(ExternalMemoryNode));
int_to_ext_node_id_map.emplace_back(current_node.lat, current_node.lon,
current_node.node_id);
ext_to_int_id_map.emplace(current_node.node_id, i);
if (current_node.barrier)
{
barrier_node_list.emplace_back(i);
}
if (current_node.traffic_lights)
{
traffic_light_node_list.emplace_back(i);
}
}
// tighten vector sizes
barrier_node_list.shrink_to_fit();
traffic_light_node_list.shrink_to_fit();
return n;
}
/**
* Reads a .osrm file and produces the edges. Edges reference nodes in the old
* OSM based format, we need to renumber it here.
*/
template <typename EdgeT>
NodeID loadEdgesFromFile(std::istream &input_stream,
const std::unordered_map<NodeID, NodeID> &ext_to_int_id_map,
std::vector<EdgeT> &edge_list)
{
EdgeWeight weight;
NodeID source, target;
unsigned nameID;
@ -419,10 +248,10 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream,
edge_list.emplace_back(source, target, nameID, weight, forward, backward, is_roundabout,
ignore_in_grid, is_access_restricted, travel_mode, is_split);
}
ext_to_int_id_map.clear();
tbb::parallel_sort(edge_list.begin(), edge_list.end());
// Removes multi-edges between nodes
for (unsigned i = 1; i < edge_list.size(); ++i)
{
if ((edge_list[i - 1].target == edge_list[i].target) &&
@ -482,7 +311,8 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream,
edge_list.erase(new_end_iter, edge_list.end()); // remove excess candidates.
edge_list.shrink_to_fit();
SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges";
return n;
return m;
}
template <typename NodeT, typename EdgeT>