From fc52dd85fea6bea68e1f6ae84a6dfdff3aea3ebc Mon Sep 17 00:00:00 2001 From: Kajari Ghosh Date: Mon, 21 Nov 2016 17:32:00 -0800 Subject: [PATCH] refactor graph_loader (#3333) Refactors the GraphLoader class to use the safer FileReader wrapper for better error handling. --- include/util/graph_loader.hpp | 64 +++++++++++++---------------------- src/extractor/extractor.cpp | 22 +++++------- src/tools/components.cpp | 15 ++++---- 3 files changed, 38 insertions(+), 63 deletions(-) diff --git a/include/util/graph_loader.hpp b/include/util/graph_loader.hpp index df0e739ff..d8437386f 100644 --- a/include/util/graph_loader.hpp +++ b/include/util/graph_loader.hpp @@ -5,6 +5,7 @@ #include "extractor/node_based_edge.hpp" #include "extractor/query_node.hpp" #include "extractor/restriction.hpp" +#include "storage/io.hpp" #include "util/exception.hpp" #include "util/fingerprint.hpp" #include "util/simple_logger.hpp" @@ -33,25 +34,14 @@ namespace util * The since the restrictions reference nodes using their external node id, * we need to renumber it to the new internal id. */ -inline unsigned loadRestrictionsFromFile(std::istream &input_stream, +inline unsigned loadRestrictionsFromFile(storage::io::FileReader &file_reader, std::vector &restriction_list) { - const FingerPrint fingerprint_valid = FingerPrint::GetValid(); - FingerPrint fingerprint_loaded; - unsigned number_of_usable_restrictions = 0; - input_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint)); - if (!fingerprint_loaded.TestContractor(fingerprint_valid)) - { - SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n" - "Reprocess to get rid of this warning."; - } - - input_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned)); + unsigned number_of_usable_restrictions = file_reader.ReadElementCount32(); restriction_list.resize(number_of_usable_restrictions); if (number_of_usable_restrictions > 0) { - input_stream.read((char *)restriction_list.data(), - number_of_usable_restrictions * sizeof(extractor::TurnRestriction)); + file_reader.ReadInto(restriction_list.data(), number_of_usable_restrictions); } return number_of_usable_restrictions; @@ -64,33 +54,24 @@ inline unsigned loadRestrictionsFromFile(std::istream &input_stream, * - nodes indexed by their internal (non-osm) id */ template -NodeID loadNodesFromFile(std::istream &input_stream, +NodeID loadNodesFromFile(storage::io::FileReader &file_reader, BarrierOutIter barriers, TrafficSignalsOutIter traffic_signals, std::vector &node_array) { - const FingerPrint fingerprint_valid = FingerPrint::GetValid(); - FingerPrint fingerprint_loaded; - input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(FingerPrint)); + NodeID number_of_nodes = file_reader.ReadElementCount32(); + SimpleLogger().Write() << "Importing number_of_nodes new = " << number_of_nodes << " nodes "; - if (!fingerprint_loaded.TestContractor(fingerprint_valid)) - { - SimpleLogger().Write(logWARNING) << ".osrm was prepared with different build.\n" - "Reprocess to get rid of this warning."; - } - - NodeID n; - input_stream.read(reinterpret_cast(&n), sizeof(NodeID)); - SimpleLogger().Write() << "Importing n = " << n << " nodes "; - - node_array.reserve(n); + node_array.resize(number_of_nodes); extractor::ExternalMemoryNode current_node; - for (NodeID i = 0; i < n; ++i) + for (NodeID i = 0; i < number_of_nodes; ++i) { - input_stream.read(reinterpret_cast(¤t_node), - sizeof(extractor::ExternalMemoryNode)); - node_array.emplace_back(current_node.lon, current_node.lat, current_node.node_id); + file_reader.ReadInto(¤t_node, 1); + + node_array[i].lon = current_node.lon; + node_array[i].lat = current_node.lat; + node_array[i].node_id = current_node.node_id; if (current_node.barrier) { @@ -105,21 +86,22 @@ NodeID loadNodesFromFile(std::istream &input_stream, } } - return n; + return number_of_nodes; } /** * Reads a .osrm file and produces the edges. */ -inline NodeID loadEdgesFromFile(std::istream &input_stream, +inline NodeID loadEdgesFromFile(storage::io::FileReader &file_reader, std::vector &edge_list) { - EdgeID m; - input_stream.read(reinterpret_cast(&m), sizeof(unsigned)); - edge_list.resize(m); - SimpleLogger().Write() << " and " << m << " edges "; + EdgeID number_of_edges = file_reader.ReadElementCount32(); + BOOST_ASSERT(sizeof(EdgeID) == sizeof(number_of_edges)); - input_stream.read((char *)edge_list.data(), m * sizeof(extractor::NodeBasedEdge)); + edge_list.resize(number_of_edges); + SimpleLogger().Write() << " and " << number_of_edges << " edges "; + + file_reader.ReadInto(edge_list.data(), number_of_edges); BOOST_ASSERT(edge_list.size() > 0); @@ -149,7 +131,7 @@ inline NodeID loadEdgesFromFile(std::istream &input_stream, SimpleLogger().Write() << "Graph loaded ok and has " << edge_list.size() << " edges"; - return m; + return number_of_edges; } } } diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index f40f00a75..41bd769ca 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -9,6 +9,7 @@ #include "extractor/scripting_environment.hpp" #include "extractor/raster_source.hpp" +#include "storage/io.hpp" #include "util/graph_loader.hpp" #include "util/io.hpp" #include "util/name_table.hpp" @@ -399,11 +400,11 @@ void Extractor::FindComponents(unsigned max_edge_id, */ std::shared_ptr Extractor::LoadRestrictionMap() { - boost::filesystem::ifstream input_stream(config.restriction_file_name, - std::ios::in | std::ios::binary); - + storage::io::FileReader file_reader(config.restriction_file_name, + storage::io::FileReader::VerifyFingerprint); std::vector restriction_list; - util::loadRestrictionsFromFile(input_stream, restriction_list); + + util::loadRestrictionsFromFile(file_reader, restriction_list); util::SimpleLogger().Write() << " - " << restriction_list.size() << " restrictions."; @@ -418,25 +419,20 @@ Extractor::LoadNodeBasedGraph(std::unordered_set &barriers, std::unordered_set &traffic_signals, std::vector &internal_to_external_node_map) { - boost::filesystem::ifstream stream(config.output_file_name, std::ios::binary); - - if (!stream) - { - throw util::exception("Unable to open " + config.output_file_name + - " trying to read the node based graph"); - } + storage::io::FileReader file_reader(config.output_file_name, + storage::io::FileReader::VerifyFingerprint); auto barriers_iter = inserter(barriers, end(barriers)); auto traffic_signals_iter = inserter(traffic_signals, end(traffic_signals)); NodeID number_of_node_based_nodes = util::loadNodesFromFile( - stream, barriers_iter, traffic_signals_iter, internal_to_external_node_map); + file_reader, barriers_iter, traffic_signals_iter, internal_to_external_node_map); util::SimpleLogger().Write() << " - " << barriers.size() << " bollard nodes, " << traffic_signals.size() << " traffic lights"; std::vector edge_list; - util::loadEdgesFromFile(stream, edge_list); + util::loadEdgesFromFile(file_reader, edge_list); if (edge_list.empty()) { diff --git a/src/tools/components.cpp b/src/tools/components.cpp index d46ccf0b7..9681d4985 100644 --- a/src/tools/components.cpp +++ b/src/tools/components.cpp @@ -50,24 +50,20 @@ void deleteFileIfExists(const std::string &file_name) } } -std::size_t loadGraph(const char *path, +std::size_t loadGraph(const std::string &path, std::vector &coordinate_list, std::vector &graph_edge_list) { - std::ifstream stream(path, std::ifstream::binary); - if (!stream) - { - throw util::exception("Cannot open osrm file"); - } + storage::io::FileReader file_reader(path, storage::io::FileReader::VerifyFingerprint); // load graph data std::vector edge_list; auto nop = boost::make_function_output_iterator([](auto) {}); - auto number_of_nodes = util::loadNodesFromFile(stream, nop, nop, coordinate_list); + auto number_of_nodes = util::loadNodesFromFile(file_reader, nop, nop, coordinate_list); - util::loadEdgesFromFile(stream, edge_list); + util::loadEdgesFromFile(file_reader, edge_list); // Building an node-based graph for (const auto &input_edge : edge_list) @@ -111,7 +107,8 @@ int main(int argc, char *argv[]) } std::vector graph_edge_list; - auto number_of_nodes = osrm::tools::loadGraph(argv[1], coordinate_list, graph_edge_list); + auto number_of_nodes = + osrm::tools::loadGraph(std::string(argv[1]), coordinate_list, graph_edge_list); tbb::parallel_sort(graph_edge_list.begin(), graph_edge_list.end()); const auto graph = std::make_shared(number_of_nodes, graph_edge_list);