Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Refactor processing_chain by splitting into sub functions

Browse Source
This commit is contained in:
Patrick Niklaus 2015-04-21 10:43:02 +02:00
parent d57f07d57e
commit 1164a65df8
9 changed files with 272 additions and 437 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -90,7 +90,7 @@ set(
add_library(COORDINATE OBJECT ${CoordinateGlob})
add_library(GITDESCRIPTION OBJECT util/git_sha.cpp)
add_library(OSRM ${OSRMSources} $<TARGET_OBJECTS:ANGLE> $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:GITDESCRIPTION> $<TARGET_OBJECTS:FINGERPRINT> $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:LOGGER> $<TARGET_OBJECTS:PHANTOMNODE> $<TARGET_OBJECTS:EXCEPTION> $<TARGET_OBJECTS:MERCATOR>)
add_library(OSRM ${OSRMSources} $<TARGET_OBJECTS:ANGLE> $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:GITDESCRIPTION> $<TARGET_OBJECTS:FINGERPRINT> $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:LOGGER> $<TARGET_OBJECTS:PHANTOMNODE> $<TARGET_OBJECTS:EXCEPTION> $<TARGET_OBJECTS:MERCATOR> $<TARGET_OBJECTS:IMPORT>)
add_library(FINGERPRINT OBJECT util/fingerprint.cpp)
add_dependencies(FINGERPRINT FingerPrintConfigure)

25
contractor/edge_based_graph_factory.cpp
View File

@ -40,21 +40,21 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <iomanip>
#include <limits>
EdgeBasedGraphFactory::EdgeBasedGraphFactory(
const std::shared_ptr<NodeBasedDynamicGraph> &node_based_graph,
std::unique_ptr<RestrictionMap> restriction_map,
std::vector<NodeID> &barrier_node_list,
std::vector<NodeID> &traffic_light_node_list,
std::vector<QueryNode> &node_info_list,
SpeedProfileProperties &speed_profile)
EdgeBasedGraphFactory::EdgeBasedGraphFactory(std::shared_ptr<NodeBasedDynamicGraph> node_based_graph,
std::shared_ptr<RestrictionMap> restriction_map,
std::unique_ptr<std::vector<NodeID>> barrier_node_list,
std::unique_ptr<std::vector<NodeID>> traffic_light_node_list,
const std::vector<QueryNode> &node_info_list,
const SpeedProfileProperties &speed_profile)
: speed_profile(speed_profile),
m_number_of_edge_based_nodes(std::numeric_limits<unsigned>::max()),
m_node_info_list(node_info_list), m_node_based_graph(node_based_graph),
m_node_info_list(node_info_list),
m_node_based_graph(std::move(node_based_graph)),
m_restriction_map(std::move(restriction_map)), max_id(0), removed_node_count(0)
{
// insert into unordered sets for fast lookup
m_barrier_nodes.insert(barrier_node_list.begin(), barrier_node_list.end());
m_traffic_lights.insert(traffic_light_node_list.begin(), traffic_light_node_list.end());
m_barrier_nodes.insert(barrier_node_list->begin(), barrier_node_list->end());
m_traffic_lights.insert(traffic_light_node_list->begin(), traffic_light_node_list->end());
}
void EdgeBasedGraphFactory::GetEdgeBasedEdges(DeallocatingVector<EdgeBasedEdge> &output_edge_list)
@ -362,11 +362,11 @@ void EdgeBasedGraphFactory::CompressGeometry()
// update any involved turn restrictions
m_restriction_map->FixupStartingTurnRestriction(node_u, node_v, node_w);
m_restriction_map->FixupArrivingTurnRestriction(node_u, node_v, node_w,
m_node_based_graph);
*m_node_based_graph);
m_restriction_map->FixupStartingTurnRestriction(node_w, node_v, node_u);
m_restriction_map->FixupArrivingTurnRestriction(node_w, node_v, node_u,
m_node_based_graph);
*m_node_based_graph);
// store compressed geometry in container
m_geometry_compressor.CompressEdge(
@ -415,6 +415,7 @@ void EdgeBasedGraphFactory::RenumberEdges()
for (const auto current_edge : m_node_based_graph->GetAdjacentEdgeRange(current_node))
{
EdgeData &edge_data = m_node_based_graph->GetEdgeData(current_edge);
// FIXME when does that happen? why can we skip here?
if (!edge_data.forward)
{
continue;

17
contractor/edge_based_graph_factory.hpp
View File

@ -59,12 +59,12 @@ class EdgeBasedGraphFactory
struct SpeedProfileProperties;
explicit EdgeBasedGraphFactory(const std::shared_ptr<NodeBasedDynamicGraph> &node_based_graph,
std::unique_ptr<RestrictionMap> restricion_map,
std::vector<NodeID> &barrier_node_list,
std::vector<NodeID> &traffic_light_node_list,
std::vector<QueryNode> &node_info_list,
SpeedProfileProperties &speed_profile);
explicit EdgeBasedGraphFactory(std::shared_ptr<NodeBasedDynamicGraph> node_based_graph,
std::shared_ptr<RestrictionMap> restricion_map,
std::unique_ptr<std::vector<NodeID>> barrier_node_list,
std::unique_ptr<std::vector<NodeID>> traffic_light_node_list,
const std::vector<QueryNode> &node_info_list,
const SpeedProfileProperties &speed_profile);
void Run(const std::string &original_edge_data_filename,
const std::string &geometry_filename,
@ -97,15 +97,16 @@ class EdgeBasedGraphFactory
unsigned m_number_of_edge_based_nodes;
std::vector<QueryNode> m_node_info_list;
std::vector<EdgeBasedNode> m_edge_based_node_list;
DeallocatingVector<EdgeBasedEdge> m_edge_based_edge_list;
const std::vector<QueryNode>& m_node_info_list;
std::shared_ptr<NodeBasedDynamicGraph> m_node_based_graph;
std::shared_ptr<RestrictionMap> m_restriction_map;
std::unordered_set<NodeID> m_barrier_nodes;
std::unordered_set<NodeID> m_traffic_lights;
std::unique_ptr<RestrictionMap> m_restriction_map;
GeometryCompressor m_geometry_compressor;

269
contractor/processing_chain.cpp
View File

@ -63,17 +63,25 @@ Prepare::~Prepare() {}
int Prepare::Process(int argc, char *argv[])
{
#ifdef WIN32
#pragma message("Memory consumption on Windows can be higher due to different bit packing")
#else
static_assert(sizeof(ImportEdge) == 20,
"changing ImportEdge type has influence on memory consumption!");
static_assert(sizeof(EdgeBasedEdge) == 16,
"changing EdgeBasedEdge type has influence on memory consumption!");
#endif
LogPolicy::GetInstance().Unmute();
TIMER_START(preparing);
TIMER_START(expansion);
if (!ParseArguments(argc, argv))
{
return 0;
}
if (!boost::filesystem::is_regular_file(input_path))
if (!boost::filesystem::is_regular_file(osrm_input_path))
{
SimpleLogger().Write(logWARNING) << "Input file " << input_path.string() << " not found!";
SimpleLogger().Write(logWARNING) << "Input file " << osrm_input_path.string() << " not found!";
return 1;
}
@ -91,7 +99,7 @@ int Prepare::Process(int argc, char *argv[])
const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
SimpleLogger().Write() << "Input file: " << input_path.filename().string();
SimpleLogger().Write() << "Input file: " << osrm_input_path.filename().string();
SimpleLogger().Write() << "Restrictions file: " << restrictions_path.filename().string();
SimpleLogger().Write() << "Profile: " << profile_path.filename().string();
SimpleLogger().Write() << "Threads: " << requested_num_threads;
@ -107,65 +115,40 @@ int Prepare::Process(int argc, char *argv[])
LogPolicy::GetInstance().Unmute();
FingerPrint fingerprint_orig;
CheckRestrictionsFile(fingerprint_orig);
boost::filesystem::ifstream input_stream(input_path, std::ios::in | std::ios::binary);
node_filename = input_path.string() + ".nodes";
edge_out = input_path.string() + ".edges";
geometry_filename = input_path.string() + ".geometry";
graph_out = input_path.string() + ".hsgr";
rtree_nodes_path = input_path.string() + ".ramIndex";
rtree_leafs_path = input_path.string() + ".fileIndex";
node_output_path = osrm_input_path.string() + ".nodes";
edge_output_path = osrm_input_path.string() + ".edges";
geometry_output_path = osrm_input_path.string() + ".geometry";
graph_output_path = osrm_input_path.string() + ".hsgr";
rtree_nodes_output_path = osrm_input_path.string() + ".ramIndex";
rtree_leafs_output_path = osrm_input_path.string() + ".fileIndex";
/*** Setup Scripting Environment ***/
// Create a new lua state
lua_State *lua_state = luaL_newstate();
// Connect LuaBind to this lua state
luabind::open(lua_state);
EdgeBasedGraphFactory::SpeedProfileProperties speed_profile;
SimpleLogger().Write() << "Generating edge-expanded graph representation";
if (!SetupScriptingEnvironment(lua_state, speed_profile))
{
return 1;
}
#ifdef WIN32
#pragma message("Memory consumption on Windows can be higher due to different bit packing")
#else
static_assert(sizeof(ImportEdge) == 20,
"changing ImportEdge type has influence on memory consumption!");
#endif
NodeID number_of_node_based_nodes = readBinaryOSRMGraphFromStream(
input_stream, edge_list, barrier_node_list, traffic_light_list,
&internal_to_external_node_map, restriction_list);
input_stream.close();
if (edge_list.empty())
{
SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
return 1;
}
SimpleLogger().Write() << restriction_list.size() << " restrictions, "
<< barrier_node_list.size() << " bollard nodes, "
<< traffic_light_list.size() << " traffic lights";
TIMER_START(expansion);
std::vector<EdgeBasedNode> node_based_edge_list;
unsigned number_of_edge_based_nodes = 0;
DeallocatingVector<EdgeBasedEdge> edge_based_edge_list;
auto internal_to_external_node_map = osrm::make_unique<std::vector<QueryNode>>();
auto graph_size =
BuildEdgeExpandedGraph(*internal_to_external_node_map,
node_based_edge_list, edge_based_edge_list);
// init node_based_edge_list, edge_based_edge_list by edgeList
number_of_edge_based_nodes =
BuildEdgeExpandedGraph(lua_state, number_of_node_based_nodes, node_based_edge_list,
edge_based_edge_list, speed_profile);
lua_close(lua_state);
auto number_of_node_based_nodes = graph_size.first;
auto number_of_edge_based_nodes = graph_size.second;
TIMER_STOP(expansion);
BuildRTree(node_based_edge_list);
SimpleLogger().Write() << "building r-tree ...";
TIMER_START(rtree);
BuildRTree(node_based_edge_list, *internal_to_external_node_map);
TIMER_STOP(rtree);
RangebasedCRC32 crc32;
if (crc32.using_hardware())
@ -182,7 +165,8 @@ int Prepare::Process(int argc, char *argv[])
node_based_edge_list.shrink_to_fit();
SimpleLogger().Write() << "CRC32: " << crc32_value;
WriteNodeMapping();
SimpleLogger().Write() << "writing node map ...";
WriteNodeMapping(std::move(internal_to_external_node_map));
/***
* Contracting the edge-expanded graph
@ -211,7 +195,7 @@ int Prepare::Process(int argc, char *argv[])
SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count
<< " edges";
boost::filesystem::ofstream hsgr_output_stream(graph_out, std::ios::binary);
boost::filesystem::ofstream hsgr_output_stream(graph_output_path, std::ios::binary);
hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint));
const unsigned max_used_node_id = 1 + [&contracted_edge_list]
{
@ -358,7 +342,7 @@ bool Prepare::ParseArguments(int argc, char *argv[])
// shown to the user
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()(
"input,i", boost::program_options::value<boost::filesystem::path>(&input_path),
"input,i", boost::program_options::value<boost::filesystem::path>(&osrm_input_path),
"Input file in .osm, .osm.bz2 or .osm.pbf format");
// positional option
@ -408,7 +392,7 @@ bool Prepare::ParseArguments(int argc, char *argv[])
if (!option_variables.count("restrictions"))
{
restrictions_path = std::string(input_path.string() + ".restrictions");
restrictions_path = std::string(osrm_input_path.string() + ".restrictions");
}
if (!option_variables.count("input"))
@ -420,36 +404,11 @@ bool Prepare::ParseArguments(int argc, char *argv[])
return true;
}
/**
\brief Loads and checks file UUIDs
*/
void Prepare::CheckRestrictionsFile(FingerPrint &fingerprint_orig)
{
boost::filesystem::ifstream restriction_stream(restrictions_path, std::ios::binary);
FingerPrint fingerprint_loaded;
unsigned number_of_usable_restrictions = 0;
restriction_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint));
if (!fingerprint_loaded.TestPrepare(fingerprint_orig))
{
SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}
restriction_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned));
restriction_list.resize(number_of_usable_restrictions);
if (number_of_usable_restrictions > 0)
{
restriction_stream.read((char *)&(restriction_list[0]),
number_of_usable_restrictions * sizeof(TurnRestriction));
}
restriction_stream.close();
}
/**
\brief Setups scripting environment (lua-scripting)
Also initializes speed profile.
*/
bool Prepare::SetupScriptingEnvironment(
void Prepare::SetupScriptingEnvironment(
lua_State *lua_state, EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile)
{
// open utility libraries string library;
@ -461,14 +420,16 @@ bool Prepare::SetupScriptingEnvironment(
// Now call our function in a lua script
if (0 != luaL_dofile(lua_state, profile_path.string().c_str()))
{
std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl;
return false;
std::stringstream msg;
msg << lua_tostring(lua_state, -1) << " occured in scripting block";
throw osrm::exception(msg.str());
}
if (0 != luaL_dostring(lua_state, "return traffic_signal_penalty\n"))
{
std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl;
return false;
std::stringstream msg;
msg << lua_tostring(lua_state, -1) << " occured in scripting block";
throw osrm::exception(msg.str());
}
speed_profile.traffic_signal_penalty = 10 * lua_tointeger(lua_state, -1);
SimpleLogger().Write(logDEBUG)
@ -476,92 +437,134 @@ bool Prepare::SetupScriptingEnvironment(
if (0 != luaL_dostring(lua_state, "return u_turn_penalty\n"))
{
std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl;
return false;
std::stringstream msg;
msg << lua_tostring(lua_state, -1) << " occured in scripting block";
throw osrm::exception(msg.str());
}
speed_profile.u_turn_penalty = 10 * lua_tointeger(lua_state, -1);
speed_profile.has_turn_penalty_function = lua_function_exists(lua_state, "turn_function");
return true;
}
/**
\brief Build load restrictions from .restriction file
*/
void Prepare::LoadRestrictionMap(const std::unordered_map<NodeID, NodeID> &external_to_internal_node_map,
RestrictionMap &restriction_map)
{
boost::filesystem::ifstream input_stream(restrictions_path, std::ios::in | std::ios::binary);
std::vector<TurnRestriction> restriction_list;
loadRestrictionsFromFile(input_stream, external_to_internal_node_map, restriction_list);
SimpleLogger().Write() << " - " << restriction_list.size() << " restrictions.";
restriction_map = RestrictionMap(restriction_list);
}
/**
\brief Build load node based graph from .osrm file and restrictions from .restrictions file
*/
std::shared_ptr<NodeBasedDynamicGraph>
Prepare::LoadNodeBasedGraph(std::vector<NodeID> &barrier_node_list,
std::vector<NodeID> &traffic_light_list,
RestrictionMap &restriction_map,
std::vector<QueryNode>& internal_to_external_node_map)
{
std::vector<ImportEdge> edge_list;
std::unordered_map<NodeID, NodeID> external_to_internal_node_map;
boost::filesystem::ifstream input_stream(osrm_input_path, std::ios::in | std::ios::binary);
NodeID number_of_node_based_nodes = loadNodesFromFile(input_stream,
barrier_node_list, traffic_light_list,
internal_to_external_node_map,
external_to_internal_node_map);
SimpleLogger().Write() << " - " << barrier_node_list.size() << " bollard nodes, "
<< traffic_light_list.size() << " traffic lights";
loadEdgesFromFile(input_stream, external_to_internal_node_map, edge_list);
LoadRestrictionMap(external_to_internal_node_map, restriction_map);
if (edge_list.empty())
{
SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
return std::shared_ptr<NodeBasedDynamicGraph>();
}
return NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list);
}
/**
\brief Building an edge-expanded graph from node-based input and turn restrictions
*/
std::size_t
Prepare::BuildEdgeExpandedGraph(lua_State *lua_state,
NodeID number_of_node_based_nodes,
std::pair<std::size_t, std::size_t>
Prepare::BuildEdgeExpandedGraph(std::vector<QueryNode> &internal_to_external_node_map,
std::vector<EdgeBasedNode> &node_based_edge_list,
DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list,
EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile)
DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list)
{
SimpleLogger().Write() << "Generating edge-expanded graph representation";
std::shared_ptr<NodeBasedDynamicGraph> node_based_graph =
NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list);
std::unique_ptr<RestrictionMap> restriction_map =
osrm::make_unique<RestrictionMap>(restriction_list);
std::shared_ptr<EdgeBasedGraphFactory> edge_based_graph_factory =
std::make_shared<EdgeBasedGraphFactory>(node_based_graph, std::move(restriction_map),
barrier_node_list, traffic_light_list,
internal_to_external_node_map, speed_profile);
edge_list.clear();
edge_list.shrink_to_fit();
lua_State *lua_state = luaL_newstate();
luabind::open(lua_state);
edge_based_graph_factory->Run(edge_out, geometry_filename, lua_state);
EdgeBasedGraphFactory::SpeedProfileProperties speed_profile;
restriction_list.clear();
restriction_list.shrink_to_fit();
barrier_node_list.clear();
barrier_node_list.shrink_to_fit();
traffic_light_list.clear();
traffic_light_list.shrink_to_fit();
SetupScriptingEnvironment(lua_state, speed_profile);
auto barrier_node_list = osrm::make_unique<std::vector<NodeID>>();
auto traffic_light_list = osrm::make_unique<std::vector<NodeID>>();
auto restriction_map = std::make_shared<RestrictionMap>();
auto node_based_graph = LoadNodeBasedGraph(*barrier_node_list, *traffic_light_list, *restriction_map, internal_to_external_node_map);
const std::size_t number_of_node_based_nodes = node_based_graph->GetNumberOfNodes();
EdgeBasedGraphFactory edge_based_graph_factory(node_based_graph,
restriction_map,
std::move(barrier_node_list),
std::move(traffic_light_list),
internal_to_external_node_map,
speed_profile);
edge_based_graph_factory.Run(edge_output_path, geometry_output_path, lua_state);
lua_close(lua_state);
const std::size_t number_of_edge_based_nodes =
edge_based_graph_factory->GetNumberOfEdgeBasedNodes();
edge_based_graph_factory.GetNumberOfEdgeBasedNodes();
BOOST_ASSERT(number_of_edge_based_nodes != std::numeric_limits<unsigned>::max());
#ifndef WIN32
static_assert(sizeof(EdgeBasedEdge) == 16,
"changing ImportEdge type has influence on memory consumption!");
#endif
edge_based_graph_factory->GetEdgeBasedEdges(edge_based_edge_list);
edge_based_graph_factory->GetEdgeBasedNodes(node_based_edge_list);
edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list);
edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list);
edge_based_graph_factory.reset();
node_based_graph.reset();
return number_of_edge_based_nodes;
return std::make_pair(number_of_node_based_nodes, number_of_edge_based_nodes);
}
/**
\brief Writing info on original (node-based) nodes
*/
void Prepare::WriteNodeMapping()
void Prepare::WriteNodeMapping(std::unique_ptr<std::vector<QueryNode>> internal_to_external_node_map)
{
SimpleLogger().Write() << "writing node map ...";
boost::filesystem::ofstream node_stream(node_filename, std::ios::binary);
const unsigned size_of_mapping = internal_to_external_node_map.size();
boost::filesystem::ofstream node_stream(node_output_path, std::ios::binary);
const unsigned size_of_mapping = internal_to_external_node_map->size();
node_stream.write((char *)&size_of_mapping, sizeof(unsigned));
if (size_of_mapping > 0)
{
node_stream.write((char *)&(internal_to_external_node_map[0]),
node_stream.write((char *) internal_to_external_node_map->data(),
size_of_mapping * sizeof(QueryNode));
}
node_stream.close();
internal_to_external_node_map.clear();
internal_to_external_node_map.shrink_to_fit();
}
/**
\brief Building rtree-based nearest-neighbor data structure
Saves info to files: '.ramIndex' and '.fileIndex'.
Saves tree into '.ramIndex' and leaves into '.fileIndex'.
*/
void Prepare::BuildRTree(std::vector<EdgeBasedNode> &node_based_edge_list)
void Prepare::BuildRTree(const std::vector<EdgeBasedNode> &node_based_edge_list, const std::vector<QueryNode>& internal_to_external_node_map)
{
SimpleLogger().Write() << "building r-tree ...";
StaticRTree<EdgeBasedNode>(node_based_edge_list, rtree_nodes_path.c_str(),
rtree_leafs_path.c_str(), internal_to_external_node_map);
StaticRTree<EdgeBasedNode>(node_based_edge_list, rtree_nodes_output_path.c_str(),
rtree_leafs_output_path.c_str(), internal_to_external_node_map);
}

44
contractor/processing_chain.hpp
View File

@ -58,38 +58,38 @@ class Prepare
protected:
bool ParseArguments(int argc, char *argv[]);
void CheckRestrictionsFile(FingerPrint &fingerprint_orig);
bool SetupScriptingEnvironment(lua_State *myLuaState,
void SetupScriptingEnvironment(lua_State *myLuaState,
EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile);
std::size_t BuildEdgeExpandedGraph(lua_State *myLuaState,
NodeID nodeBasedNodeNumber,
std::vector<EdgeBasedNode> &nodeBasedEdgeList,
DeallocatingVector<EdgeBasedEdge> &edgeBasedEdgeList,
EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile);
void WriteNodeMapping();
void BuildRTree(std::vector<EdgeBasedNode> &node_based_edge_list);
void LoadRestrictionMap(const std::unordered_map<NodeID, NodeID> &external_to_internal_node_map,
RestrictionMap &restriction_map);
std::shared_ptr<NodeBasedDynamicGraph>
LoadNodeBasedGraph(std::vector<NodeID> &barrier_node_list,
std::vector<NodeID> &traffic_light_list,
RestrictionMap &restriction_map,
std::vector<QueryNode>& internal_to_external_node_map);
std::pair<std::size_t, std::size_t>
BuildEdgeExpandedGraph(std::vector<QueryNode> &internal_to_external_node_map,
std::vector<EdgeBasedNode> &node_based_edge_list,
DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list);
void WriteNodeMapping(std::unique_ptr<std::vector<QueryNode>> internal_to_external_node_map);
void BuildRTree(const std::vector<EdgeBasedNode> &node_based_edge_list,
const std::vector<QueryNode> &internal_to_external_node_map);
private:
std::vector<QueryNode> internal_to_external_node_map;
std::vector<TurnRestriction> restriction_list;
std::vector<NodeID> barrier_node_list;
std::vector<NodeID> traffic_light_list;
std::vector<ImportEdge> edge_list;
unsigned requested_num_threads;
boost::filesystem::path config_file_path;
boost::filesystem::path input_path;
boost::filesystem::path osrm_input_path;
boost::filesystem::path restrictions_path;
boost::filesystem::path preinfo_path;
boost::filesystem::path profile_path;
std::string node_filename;
std::string edge_out;
std::string info_out;
std::string geometry_filename;
std::string graph_out;
std::string rtree_nodes_path;
std::string rtree_leafs_path;
std::string node_output_path;
std::string edge_output_path;
std::string geometry_output_path;
std::string graph_output_path;
std::string rtree_nodes_output_path;
std::string rtree_leafs_output_path;
};
#endif // PROCESSING_CHAIN_HPP

3
data_structures/node_based_graph.hpp
View File

@ -186,9 +186,8 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector<ImportEdge
SimpleLogger().Write() << "merged " << edges_list.size() - edge_count << " edges out of "
<< edges_list.size();
auto graph = std::make_shared<NodeBasedDynamicGraph>(
return std::make_shared<NodeBasedDynamicGraph>(
static_cast<NodeBasedDynamicGraph::NodeIterator>(number_of_nodes), edges_list);
return graph;
}
#endif // NODE_BASED_GRAPH_HPP

7
data_structures/restriction_map.hpp
View File

@ -91,6 +91,7 @@ template <> struct hash<RestrictionTarget>
class RestrictionMap
{
public:
RestrictionMap() = default;
RestrictionMap(const std::vector<TurnRestriction> &restriction_list);
// Replace end v with w in each turn restriction containing u as via node
@ -98,7 +99,7 @@ class RestrictionMap
void FixupArrivingTurnRestriction(const NodeID node_u,
const NodeID node_v,
const NodeID node_w,
const std::shared_ptr<GraphT> &graph)
const GraphT &graph)
{
BOOST_ASSERT(node_u != SPECIAL_NODEID);
BOOST_ASSERT(node_v != SPECIAL_NODEID);
@ -112,9 +113,9 @@ class RestrictionMap
// find all potential start edges. It is more efficent to get a (small) list
// of potential start edges than iterating over all buckets
std::vector<NodeID> predecessors;
for (const EdgeID current_edge_id : graph->GetAdjacentEdgeRange(node_u))
for (const EdgeID current_edge_id : graph.GetAdjacentEdgeRange(node_u))
{
const NodeID target = graph->GetTarget(current_edge_id);
const NodeID target = graph.GetTarget(current_edge_id);
if (node_v != target)
{
predecessors.push_back(target);

2
data_structures/static_rtree.hpp
View File

@ -344,7 +344,7 @@ class StaticRTree
StaticRTree(const StaticRTree &) = delete;
// Construct a packed Hilbert-R-Tree with Kamel-Faloutsos algorithm [1]
explicit StaticRTree(std::vector<EdgeDataT> &input_data_vector,
explicit StaticRTree(const std::vector<EdgeDataT> &input_data_vector,
const std::string tree_node_filename,
const std::string leaf_node_filename,
const std::vector<QueryNode> &coordinate_list)

340
util/graph_loader.hpp
View File

@ -52,269 +52,34 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <unordered_map>
#include <vector>
template <typename EdgeT>
NodeID read_undirected_osrm_stream(std::istream &input_stream,
std::vector<EdgeT> &edge_list,
std::vector<FixedPointCoordinate> &coordinate_list)
/**
* Reads the .restrictions file and loads it to a vector.
* The since the restrictions reference nodes using their external node id,
* we need to renumber it to the new internal id.
*/
unsigned loadRestrictionsFromFile(std::istream& input_stream,
const std::unordered_map<NodeID, NodeID>& ext_to_int_id_map,
std::vector<TurnRestriction>& restriction_list)
{
const FingerPrint fingerprint_orig;
FingerPrint fingerprint_loaded;
input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(FingerPrint));
if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig))
unsigned number_of_usable_restrictions = 0;
input_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint));
if (!fingerprint_loaded.TestPrepare(fingerprint_orig))
{
SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n"
SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}
std::unordered_map<NodeID, NodeID> ext_to_int_id_map;
NodeID n;
input_stream.read(reinterpret_cast<char *>(&n), sizeof(NodeID));
SimpleLogger().Write() << "Importing n = " << n << " nodes ";
ExternalMemoryNode current_node;
for (NodeID i = 0; i < n; ++i)
input_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned));
restriction_list.resize(number_of_usable_restrictions);
if (number_of_usable_restrictions > 0)
{
input_stream.read(reinterpret_cast<char *>(&current_node), sizeof(ExternalMemoryNode));
coordinate_list.emplace_back(current_node.lat, current_node.lon);
ext_to_int_id_map.emplace(current_node.node_id, i);
// if (current_node.barrier)
// {
// barrier_node_list.emplace_back(i);
// }
// if (current_node.traffic_lights)
// {
// traffic_light_node_list.emplace_back(i);
// }
input_stream.read((char *) restriction_list.data(),
number_of_usable_restrictions * sizeof(TurnRestriction));
}
// tighten vector sizes
// barrier_node_list.shrink_to_fit();
// traffic_light_node_list.shrink_to_fit();
// renumber nodes in turn restrictions
// for (TurnRestriction &current_restriction : restriction_list)
// {
// auto internal_id_iter = ext_to_int_id_map.find(current_restriction.from.node);
// if (internal_id_iter == ext_to_int_id_map.end())
// {
// SimpleLogger().Write(logDEBUG) << "Unmapped from node " <<
// current_restriction.from.node
// << " of restriction";
// continue;
// }
// current_restriction.from.node = internal_id_iter->second;
// internal_id_iter = ext_to_int_id_map.find(current_restriction.via.node);
// if (internal_id_iter == ext_to_int_id_map.end())
// {
// SimpleLogger().Write(logDEBUG) << "Unmapped via node " <<
// current_restriction.via.node
// << " of restriction";
// continue;
// }
// current_restriction.via.node = internal_id_iter->second;
// internal_id_iter = ext_to_int_id_map.find(current_restriction.to.node);
// if (internal_id_iter == ext_to_int_id_map.end())
// {
// SimpleLogger().Write(logDEBUG) << "Unmapped to node " << current_restriction.to.node
// << " of restriction";
// continue;
// }
// current_restriction.to.node = internal_id_iter->second;
// }
EdgeWeight weight;
NodeID source, target;
unsigned nameID;
int length;
short dir; // direction (0 = open, 1 = forward, 2+ = open)
bool is_roundabout, ignore_in_grid, is_access_restricted, is_split;
TravelMode travel_mode;
EdgeID m;
input_stream.read(reinterpret_cast<char *>(&m), sizeof(unsigned));
edge_list.reserve(m);
SimpleLogger().Write() << " and " << m << " edges ";
for (EdgeID i = 0; i < m; ++i)
{
input_stream.read(reinterpret_cast<char *>(&source), sizeof(unsigned));
input_stream.read(reinterpret_cast<char *>(&target), sizeof(unsigned));
input_stream.read(reinterpret_cast<char *>(&length), sizeof(int));
input_stream.read(reinterpret_cast<char *>(&dir), sizeof(short));
input_stream.read(reinterpret_cast<char *>(&weight), sizeof(int));
input_stream.read(reinterpret_cast<char *>(&nameID), sizeof(unsigned));
input_stream.read(reinterpret_cast<char *>(&is_roundabout), sizeof(bool));
input_stream.read(reinterpret_cast<char *>(&ignore_in_grid), sizeof(bool));
input_stream.read(reinterpret_cast<char *>(&is_access_restricted), sizeof(bool));
input_stream.read(reinterpret_cast<char *>(&travel_mode), sizeof(TravelMode));
input_stream.read(reinterpret_cast<char *>(&is_split), sizeof(bool));
BOOST_ASSERT_MSG(length > 0, "loaded null length edge");
BOOST_ASSERT_MSG(weight > 0, "loaded null weight");
BOOST_ASSERT_MSG(0 <= dir && dir <= 2, "loaded bogus direction");
// bool forward = true;
// bool backward = true;
// if (1 == dir)
// {
// backward = false;
// }
// if (2 == dir)
// {
// forward = false;
// }
// translate the external NodeIDs to internal IDs
auto internal_id_iter = ext_to_int_id_map.find(source);
if (ext_to_int_id_map.find(source) == ext_to_int_id_map.end())
{
#ifndef NDEBUG
SimpleLogger().Write(logWARNING) << " unresolved source NodeID: " << source;
#endif
continue;
}
source = internal_id_iter->second;
internal_id_iter = ext_to_int_id_map.find(target);
if (ext_to_int_id_map.find(target) == ext_to_int_id_map.end())
{
#ifndef NDEBUG
SimpleLogger().Write(logWARNING) << "unresolved target NodeID : " << target;
#endif
continue;
}
target = internal_id_iter->second;
BOOST_ASSERT_MSG(source != SPECIAL_NODEID && target != SPECIAL_NODEID,
"nonexisting source or target");
if (source > target)
{
std::swap(source, target);
// std::swap(forward, backward);
}
edge_list.emplace_back(source, target);
}
ext_to_int_id_map.clear();
tbb::parallel_sort(edge_list.begin(), edge_list.end());
// for (unsigned i = 1; i < edge_list.size(); ++i)
// {
// if ((edge_list[i - 1].target == edge_list[i].target) &&
// (edge_list[i - 1].source == edge_list[i].source))
// {
// const bool edge_flags_equivalent = (edge_list[i - 1].forward == edge_list[i].forward)
// &&
// (edge_list[i - 1].backward ==
// edge_list[i].backward);
// const bool edge_flags_are_superset1 =
// (edge_list[i - 1].forward && edge_list[i - 1].backward) &&
// (edge_list[i].forward != edge_list[i].backward);
// const bool edge_flags_are_superset_2 =
// (edge_list[i].forward && edge_list[i].backward) &&
// (edge_list[i - 1].forward != edge_list[i - 1].backward);
// if (edge_flags_equivalent)
// {
// edge_list[i].weight = std::min(edge_list[i - 1].weight, edge_list[i].weight);
// edge_list[i - 1].source = SPECIAL_NODEID;
// }
// else if (edge_flags_are_superset1)
// {
// if (edge_list[i - 1].weight <= edge_list[i].weight)
// {
// // edge i-1 is smaller and goes in both directions. Throw away the other edge
// edge_list[i].source = SPECIAL_NODEID;
// }
// else
// {
// // edge i-1 is open in both directions, but edge i is smaller in one
// direction.
// // Close edge i-1 in this direction
// edge_list[i - 1].forward = !edge_list[i].forward;
// edge_list[i - 1].backward = !edge_list[i].backward;
// }
// }
// else if (edge_flags_are_superset_2)
// {
// if (edge_list[i - 1].weight <= edge_list[i].weight)
// {
// // edge i-1 is smaller for one direction. edge i is open in both. close edge
// i
// // in the other direction
// edge_list[i].forward = !edge_list[i - 1].forward;
// edge_list[i].backward = !edge_list[i - 1].backward;
// }
// else
// {
// // edge i is smaller and goes in both direction. Throw away edge i-1
// edge_list[i - 1].source = SPECIAL_NODEID;
// }
// }
// }
// }
// const auto new_end_iter =
// std::remove_if(edge_list.begin(), edge_list.end(), [](const EdgeT &edge)
// {
// return edge.source == SPECIAL_NODEID || edge.target == SPECIAL_NODEID;
// });
// edge_list.erase(new_end_iter, edge_list.end()); // remove excess candidates.
// edge_list.shrink_to_fit();
SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges";
return n;
}
template <typename EdgeT>
NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream,
std::vector<EdgeT> &edge_list,
std::vector<NodeID> &barrier_node_list,
std::vector<NodeID> &traffic_light_node_list,
std::vector<QueryNode> *int_to_ext_node_id_map,
std::vector<TurnRestriction> &restriction_list)
{
const FingerPrint fingerprint_orig;
FingerPrint fingerprint_loaded;
input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(FingerPrint));
if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig))
{
SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}
std::unordered_map<NodeID, NodeID> ext_to_int_id_map;
NodeID n;
input_stream.read(reinterpret_cast<char *>(&n), sizeof(NodeID));
SimpleLogger().Write() << "Importing n = " << n << " nodes ";
ExternalMemoryNode current_node;
for (NodeID i = 0; i < n; ++i)
{
input_stream.read(reinterpret_cast<char *>(&current_node), sizeof(ExternalMemoryNode));
int_to_ext_node_id_map->emplace_back(current_node.lat, current_node.lon,
current_node.node_id);
ext_to_int_id_map.emplace(current_node.node_id, i);
if (current_node.barrier)
{
barrier_node_list.emplace_back(i);
}
if (current_node.traffic_lights)
{
traffic_light_node_list.emplace_back(i);
}
}
// tighten vector sizes
barrier_node_list.shrink_to_fit();
traffic_light_node_list.shrink_to_fit();
// renumber nodes in turn restrictions
// renumber ids referenced in restrictions
for (TurnRestriction &current_restriction : restriction_list)
{
auto internal_id_iter = ext_to_int_id_map.find(current_restriction.from.node);
@ -346,6 +111,70 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream,
current_restriction.to.node = internal_id_iter->second;
}
return number_of_usable_restrictions;
}
/**
* Reads the beginning of an .osrm file and produces:
* - list of barrier nodes
* - list of traffic lights
* - index to original node id map
* - original node id to index map
*/
NodeID loadNodesFromFile(std::istream &input_stream,
std::vector<NodeID> &barrier_node_list,
std::vector<NodeID> &traffic_light_node_list,
std::vector<QueryNode> &int_to_ext_node_id_map,
std::unordered_map<NodeID, NodeID> &ext_to_int_id_map)
{
const FingerPrint fingerprint_orig;
FingerPrint fingerprint_loaded;
input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(FingerPrint));
if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig))
{
SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}
NodeID n;
input_stream.read(reinterpret_cast<char *>(&n), sizeof(NodeID));
SimpleLogger().Write() << "Importing n = " << n << " nodes ";
ExternalMemoryNode current_node;
for (NodeID i = 0; i < n; ++i)
{
input_stream.read(reinterpret_cast<char *>(&current_node), sizeof(ExternalMemoryNode));
int_to_ext_node_id_map.emplace_back(current_node.lat, current_node.lon,
current_node.node_id);
ext_to_int_id_map.emplace(current_node.node_id, i);
if (current_node.barrier)
{
barrier_node_list.emplace_back(i);
}
if (current_node.traffic_lights)
{
traffic_light_node_list.emplace_back(i);
}
}
// tighten vector sizes
barrier_node_list.shrink_to_fit();
traffic_light_node_list.shrink_to_fit();
return n;
}
/**
* Reads a .osrm file and produces the edges. Edges reference nodes in the old
* OSM based format, we need to renumber it here.
*/
template <typename EdgeT>
NodeID loadEdgesFromFile(std::istream &input_stream,
const std::unordered_map<NodeID, NodeID> &ext_to_int_id_map,
std::vector<EdgeT> &edge_list)
{
EdgeWeight weight;
NodeID source, target;
unsigned nameID;
@ -419,10 +248,10 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream,
edge_list.emplace_back(source, target, nameID, weight, forward, backward, is_roundabout,
ignore_in_grid, is_access_restricted, travel_mode, is_split);
}
ext_to_int_id_map.clear();
tbb::parallel_sort(edge_list.begin(), edge_list.end());
// Removes multi-edges between nodes
for (unsigned i = 1; i < edge_list.size(); ++i)
{
if ((edge_list[i - 1].target == edge_list[i].target) &&
@ -482,7 +311,8 @@ NodeID readBinaryOSRMGraphFromStream(std::istream &input_stream,
edge_list.erase(new_end_iter, edge_list.end()); // remove excess candidates.
edge_list.shrink_to_fit();
SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges";
return n;
return m;
}
template <typename NodeT, typename EdgeT>