Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

refactor graph_loader (#3333)

Browse Source 
Refactors the GraphLoader class to use the safer FileReader wrapper for better error handling.
This commit is contained in:
Kajari Ghosh 2016-11-21 17:32:00 -08:00 committed by Daniel Patterson
parent 6d961d4a15
commit fc52dd85fe
3 changed files with 38 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
include/util/graph_loader.hpp
View File

@ -5,6 +5,7 @@
#include "extractor/node_based_edge.hpp"
#include "extractor/query_node.hpp"
#include "extractor/restriction.hpp"
#include "storage/io.hpp"
#include "util/exception.hpp"
#include "util/fingerprint.hpp"
#include "util/simple_logger.hpp"
@ -33,25 +34,14 @@ namespace util
* The since the restrictions reference nodes using their external node id,
* we need to renumber it to the new internal id.
*/
inline unsigned loadRestrictionsFromFile(std::istream &input_stream,
inline unsigned loadRestrictionsFromFile(storage::io::FileReader &file_reader,
std::vector<extractor::TurnRestriction> &restriction_list)
{
const FingerPrint fingerprint_valid = FingerPrint::GetValid();
FingerPrint fingerprint_loaded;
unsigned number_of_usable_restrictions = 0;
input_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint));
if (!fingerprint_loaded.TestContractor(fingerprint_valid))
{
SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}
input_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned));
unsigned number_of_usable_restrictions = file_reader.ReadElementCount32();
restriction_list.resize(number_of_usable_restrictions);
if (number_of_usable_restrictions > 0)
{
input_stream.read((char *)restriction_list.data(),
number_of_usable_restrictions * sizeof(extractor::TurnRestriction));
file_reader.ReadInto(restriction_list.data(), number_of_usable_restrictions);
}
return number_of_usable_restrictions;
@ -64,33 +54,24 @@ inline unsigned loadRestrictionsFromFile(std::istream &input_stream,
* - nodes indexed by their internal (non-osm) id
*/
template <typename BarrierOutIter, typename TrafficSignalsOutIter>
NodeID loadNodesFromFile(std::istream &input_stream,
NodeID loadNodesFromFile(storage::io::FileReader &file_reader,
BarrierOutIter barriers,
TrafficSignalsOutIter traffic_signals,
std::vector<extractor::QueryNode> &node_array)
{
const FingerPrint fingerprint_valid = FingerPrint::GetValid();
FingerPrint fingerprint_loaded;
input_stream.read(reinterpret_cast<char *>(&fingerprint_loaded), sizeof(FingerPrint));
NodeID number_of_nodes = file_reader.ReadElementCount32();
SimpleLogger().Write() << "Importing number_of_nodes new = " << number_of_nodes << " nodes ";
if (!fingerprint_loaded.TestContractor(fingerprint_valid))
{
SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}
NodeID n;
input_stream.read(reinterpret_cast<char *>(&n), sizeof(NodeID));
SimpleLogger().Write() << "Importing n = " << n << " nodes ";
node_array.reserve(n);
node_array.resize(number_of_nodes);
extractor::ExternalMemoryNode current_node;
for (NodeID i = 0; i < n; ++i)
for (NodeID i = 0; i < number_of_nodes; ++i)
{
input_stream.read(reinterpret_cast<char *>(&current_node),
sizeof(extractor::ExternalMemoryNode));
node_array.emplace_back(current_node.lon, current_node.lat, current_node.node_id);
file_reader.ReadInto(&current_node, 1);
node_array[i].lon = current_node.lon;
node_array[i].lat = current_node.lat;
node_array[i].node_id = current_node.node_id;
if (current_node.barrier)
{
@ -105,21 +86,22 @@ NodeID loadNodesFromFile(std::istream &input_stream,
}
}
return n;
return number_of_nodes;
}
/**
* Reads a .osrm file and produces the edges.
*/
inline NodeID loadEdgesFromFile(std::istream &input_stream,
inline NodeID loadEdgesFromFile(storage::io::FileReader &file_reader,
std::vector<extractor::NodeBasedEdge> &edge_list)
{
EdgeID m;
input_stream.read(reinterpret_cast<char *>(&m), sizeof(unsigned));
edge_list.resize(m);
SimpleLogger().Write() << " and " << m << " edges ";
EdgeID number_of_edges = file_reader.ReadElementCount32();
BOOST_ASSERT(sizeof(EdgeID) == sizeof(number_of_edges));
input_stream.read((char *)edge_list.data(), m * sizeof(extractor::NodeBasedEdge));
edge_list.resize(number_of_edges);
SimpleLogger().Write() << " and " << number_of_edges << " edges ";
file_reader.ReadInto(edge_list.data(), number_of_edges);
BOOST_ASSERT(edge_list.size() > 0);
@ -149,7 +131,7 @@ inline NodeID loadEdgesFromFile(std::istream &input_stream,
SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges";
return m;
return number_of_edges;
}
}
}

22
src/extractor/extractor.cpp
View File

@ -9,6 +9,7 @@
#include "extractor/scripting_environment.hpp"
#include "extractor/raster_source.hpp"
#include "storage/io.hpp"
#include "util/graph_loader.hpp"
#include "util/io.hpp"
#include "util/name_table.hpp"
@ -399,11 +400,11 @@ void Extractor::FindComponents(unsigned max_edge_id,
*/
std::shared_ptr<RestrictionMap> Extractor::LoadRestrictionMap()
{
boost::filesystem::ifstream input_stream(config.restriction_file_name,
std::ios::in | std::ios::binary);
storage::io::FileReader file_reader(config.restriction_file_name,
storage::io::FileReader::VerifyFingerprint);
std::vector<TurnRestriction> restriction_list;
util::loadRestrictionsFromFile(input_stream, restriction_list);
util::loadRestrictionsFromFile(file_reader, restriction_list);
util::SimpleLogger().Write() << " - " << restriction_list.size() << " restrictions.";
@ -418,25 +419,20 @@ Extractor::LoadNodeBasedGraph(std::unordered_set<NodeID> &barriers,
std::unordered_set<NodeID> &traffic_signals,
std::vector<QueryNode> &internal_to_external_node_map)
{
boost::filesystem::ifstream stream(config.output_file_name, std::ios::binary);
if (!stream)
{
throw util::exception("Unable to open " + config.output_file_name +
" trying to read the node based graph");
}
storage::io::FileReader file_reader(config.output_file_name,
storage::io::FileReader::VerifyFingerprint);
auto barriers_iter = inserter(barriers, end(barriers));
auto traffic_signals_iter = inserter(traffic_signals, end(traffic_signals));
NodeID number_of_node_based_nodes = util::loadNodesFromFile(
stream, barriers_iter, traffic_signals_iter, internal_to_external_node_map);
file_reader, barriers_iter, traffic_signals_iter, internal_to_external_node_map);
util::SimpleLogger().Write() << " - " << barriers.size() << " bollard nodes, "
<< traffic_signals.size() << " traffic lights";
std::vector<NodeBasedEdge> edge_list;
util::loadEdgesFromFile(stream, edge_list);
util::loadEdgesFromFile(file_reader, edge_list);
if (edge_list.empty())
{

15
src/tools/components.cpp
View File

@ -50,24 +50,20 @@ void deleteFileIfExists(const std::string &file_name)
}
}
std::size_t loadGraph(const char *path,
std::size_t loadGraph(const std::string &path,
std::vector<extractor::QueryNode> &coordinate_list,
std::vector<TarjanEdge> &graph_edge_list)
{
std::ifstream stream(path, std::ifstream::binary);
if (!stream)
{
throw util::exception("Cannot open osrm file");
}
storage::io::FileReader file_reader(path, storage::io::FileReader::VerifyFingerprint);
// load graph data
std::vector<extractor::NodeBasedEdge> edge_list;
auto nop = boost::make_function_output_iterator([](auto) {});
auto number_of_nodes = util::loadNodesFromFile(stream, nop, nop, coordinate_list);
auto number_of_nodes = util::loadNodesFromFile(file_reader, nop, nop, coordinate_list);
util::loadEdgesFromFile(stream, edge_list);
util::loadEdgesFromFile(file_reader, edge_list);
// Building an node-based graph
for (const auto &input_edge : edge_list)
@ -111,7 +107,8 @@ int main(int argc, char *argv[])
}
std::vector<osrm::tools::TarjanEdge> graph_edge_list;
auto number_of_nodes = osrm::tools::loadGraph(argv[1], coordinate_list, graph_edge_list);
auto number_of_nodes =
osrm::tools::loadGraph(std::string(argv[1]), coordinate_list, graph_edge_list);
tbb::parallel_sort(graph_edge_list.begin(), graph_edge_list.end());
const auto graph = std::make_shared<osrm::tools::TarjanGraph>(number_of_nodes, graph_edge_list);