From 7b97e1035f78fc7834b4433172a7e32bdf329219 Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Tue, 18 Oct 2016 12:00:02 -0700 Subject: [PATCH] refactor loading code of .nodes and .edges files and move to io.hpp --- .../engine/datafacade/internal_datafacade.hpp | 58 +++++++------- include/storage/io.hpp | 78 +++++++++++++++++-- src/storage/storage.cpp | 49 +++++------- 3 files changed, 118 insertions(+), 67 deletions(-) diff --git a/include/engine/datafacade/internal_datafacade.hpp b/include/engine/datafacade/internal_datafacade.hpp index 7ed9637f2..a6044fdab 100644 --- a/include/engine/datafacade/internal_datafacade.hpp +++ b/include/engine/datafacade/internal_datafacade.hpp @@ -181,27 +181,27 @@ class InternalDataFacade final : public BaseDataFacade util::SimpleLogger().Write() << "Data checksum is " << m_check_sum; } - void LoadNodeAndEdgeInformation(const boost::filesystem::path &nodes_file, - const boost::filesystem::path &edges_file) + void LoadNodeAndEdgeInformation(const boost::filesystem::path &nodes_file_path, + const boost::filesystem::path &edges_file_path) { - boost::filesystem::ifstream nodes_input_stream(nodes_file, std::ios::binary); - - extractor::QueryNode current_node; - unsigned number_of_coordinates = 0; - nodes_input_stream.read((char *)&number_of_coordinates, sizeof(unsigned)); - m_coordinate_list.resize(number_of_coordinates); - m_osmnodeid_list.reserve(number_of_coordinates); - for (unsigned i = 0; i < number_of_coordinates; ++i) + boost::filesystem::ifstream nodes_input_stream(nodes_file_path, std::ios::binary); + if (!nodes_input_stream) { - nodes_input_stream.read((char *)¤t_node, sizeof(extractor::QueryNode)); - m_coordinate_list[i] = util::Coordinate(current_node.lon, current_node.lat); - m_osmnodeid_list.push_back(current_node.node_id); - BOOST_ASSERT(m_coordinate_list[i].IsValid()); + throw util::exception("Could not open " + nodes_file_path.string() + " for reading."); } - boost::filesystem::ifstream edges_input_stream(edges_file, std::ios::binary); - unsigned number_of_edges = 0; - edges_input_stream.read((char *)&number_of_edges, sizeof(unsigned)); + std::uint32_t number_of_coordinates = storage::io::readNodesSize(nodes_input_stream); + m_coordinate_list.resize(number_of_coordinates); + m_osmnodeid_list.reserve(number_of_coordinates); + storage::io::readNodesData( + nodes_input_stream, m_coordinate_list.data(), m_osmnodeid_list, number_of_coordinates); + + boost::filesystem::ifstream edges_input_stream(edges_file_path, std::ios::binary); + if (!edges_input_stream) + { + throw util::exception("Could not open " + edges_file_path.string() + " for reading."); + } + auto number_of_edges = storage::io::readEdgesSize(edges_input_stream); m_via_geometry_list.resize(number_of_edges); m_name_ID_list.resize(number_of_edges); m_turn_instruction_list.resize(number_of_edges); @@ -211,20 +211,16 @@ class InternalDataFacade final : public BaseDataFacade m_pre_turn_bearing.resize(number_of_edges); m_post_turn_bearing.resize(number_of_edges); - extractor::OriginalEdgeData current_edge_data; - for (unsigned i = 0; i < number_of_edges; ++i) - { - edges_input_stream.read((char *)&(current_edge_data), - sizeof(extractor::OriginalEdgeData)); - m_via_geometry_list[i] = current_edge_data.via_geometry; - m_name_ID_list[i] = current_edge_data.name_id; - m_turn_instruction_list[i] = current_edge_data.turn_instruction; - m_lane_data_id[i] = current_edge_data.lane_data_id; - m_travel_mode_list[i] = current_edge_data.travel_mode; - m_entry_class_id_list[i] = current_edge_data.entry_classid; - m_pre_turn_bearing[i] = current_edge_data.pre_turn_bearing; - m_post_turn_bearing[i] = current_edge_data.post_turn_bearing; - } + storage::io::readEdgesData(edges_input_stream, + m_via_geometry_list.data(), + m_name_ID_list.data(), + m_turn_instruction_list.data(), + m_lane_data_id.data(), + m_travel_mode_list.data(), + m_entry_class_id_list.data(), + m_pre_turn_bearing.data(), + m_post_turn_bearing.data(), + number_of_edges); } void LoadCoreInformation(const boost::filesystem::path &core_data_file) diff --git a/include/storage/io.hpp b/include/storage/io.hpp index edaab5b54..f2e8c54a1 100644 --- a/include/storage/io.hpp +++ b/include/storage/io.hpp @@ -63,9 +63,9 @@ inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream) // Needs to be called after readHSGRHeader() to get the correct offset in the stream template void readHSGR(boost::filesystem::ifstream &input_stream, - NodeT *node_buffer, + NodeT node_buffer[], std::uint32_t number_of_nodes, - EdgeT *edge_buffer, + EdgeT edge_buffer[], std::uint32_t number_of_edges) { input_stream.read(reinterpret_cast(node_buffer), number_of_nodes * sizeof(NodeT)); @@ -74,20 +74,88 @@ void readHSGR(boost::filesystem::ifstream &input_stream, // Returns the size of the timestamp in a file inline std::uint32_t readTimestampSize(boost::filesystem::ifstream ×tamp_input_stream) { - timestamp_input_stream.seekg (0, timestamp_input_stream.end); + timestamp_input_stream.seekg(0, timestamp_input_stream.end); auto length = timestamp_input_stream.tellg(); - timestamp_input_stream.seekg (0, timestamp_input_stream.beg); + timestamp_input_stream.seekg(0, timestamp_input_stream.beg); return length; } // Reads the timestamp in a file inline void readTimestamp(boost::filesystem::ifstream ×tamp_input_stream, - char *timestamp, + char timestamp[], std::size_t timestamp_length) { timestamp_input_stream.read(timestamp, timestamp_length * sizeof(char)); } +// Returns the number of edges in a .edges file +inline std::uint32_t readEdgesSize(boost::filesystem::ifstream &edges_input_stream) +{ + std::uint32_t number_of_edges; + edges_input_stream.read((char *)&number_of_edges, sizeof(std::uint32_t)); + return number_of_edges; +} + +// Reads edge data from .edge files which includes its +// geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID +template +void readEdgesData(boost::filesystem::ifstream &edges_input_stream, + GeometryIDT geometry_list[], + NameIDT name_id_list[], + TurnInstructionT turn_instruction_list[], + LaneDataIDT lane_data_id_list[], + TravelModeT travel_mode_list[], + EntryClassIDT entry_class_id_list[], + PreTurnBearingT pre_turn_bearing_list[], + PostTurnBearingT post_turn_bearing_list[], + std::uint32_t number_of_edges) +{ + extractor::OriginalEdgeData current_edge_data; + for (std::uint32_t i = 0; i < number_of_edges; ++i) + { + edges_input_stream.read((char *)&(current_edge_data), sizeof(extractor::OriginalEdgeData)); + geometry_list[i] = current_edge_data.via_geometry; + name_id_list[i] = current_edge_data.name_id; + turn_instruction_list[i] = current_edge_data.turn_instruction; + lane_data_id_list[i] = current_edge_data.lane_data_id; + travel_mode_list[i] = current_edge_data.travel_mode; + entry_class_id_list[i] = current_edge_data.entry_classid; + pre_turn_bearing_list[i] = current_edge_data.pre_turn_bearing; + post_turn_bearing_list[i] = current_edge_data.post_turn_bearing; + } +} + +// Returns the number of nodes in a .nodes file +inline std::uint32_t readNodesSize(boost::filesystem::ifstream &nodes_input_stream) +{ + std::uint32_t number_of_coordinates; + nodes_input_stream.read((char *)&number_of_coordinates, sizeof(std::uint32_t)); + return number_of_coordinates; +} + +// Reads coordinates and OSM node IDs from .nodes files +template +void readNodesData(boost::filesystem::ifstream &nodes_input_stream, + CoordinateT coordinate_list[], + OSMNodeIDVectorT &osmnodeid_list, + std::uint32_t number_of_coordinates) +{ + extractor::QueryNode current_node; + for (std::uint32_t i = 0; i < number_of_coordinates; ++i) + { + nodes_input_stream.read((char *)¤t_node, sizeof(extractor::QueryNode)); + coordinate_list[i] = CoordinateT(current_node.lon, current_node.lat); + osmnodeid_list.push_back(current_node.node_id); + BOOST_ASSERT(coordinate_list[i].IsValid()); + } +} } } } diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 518bca697..b1352bd55 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -1,10 +1,3 @@ -#include "storage/storage.hpp" - -#include "storage/io.hpp" -#include "storage/shared_barriers.hpp" -#include "storage/shared_datatype.hpp" -#include "storage/shared_memory.hpp" - #include "contractor/query_edge.hpp" #include "extractor/compressed_edge_container.hpp" #include "extractor/guidance/turn_instruction.hpp" @@ -12,6 +5,11 @@ #include "extractor/profile_properties.hpp" #include "extractor/query_node.hpp" #include "extractor/travel_mode.hpp" +#include "storage/io.hpp" +#include "storage/shared_barriers.hpp" +#include "storage/shared_datatype.hpp" +#include "storage/shared_memory.hpp" +#include "storage/storage.hpp" #include "engine/datafacade/datafacade_base.hpp" #include "util/coordinate.hpp" #include "util/exception.hpp" @@ -232,8 +230,7 @@ Storage::ReturnCode Storage::Run(int max_wait) throw util::exception("Could not open " + config.edges_data_path.string() + " for reading."); } - unsigned number_of_original_edges = 0; - edges_input_stream.read((char *)&number_of_original_edges, sizeof(unsigned)); + auto number_of_original_edges = io::readEdgesSize(edges_input_stream); // note: settings this all to the same size is correct, we extract them from the same struct shared_layout_ptr->SetBlockSize(SharedDataLayout::VIA_NODE_LIST, @@ -305,8 +302,7 @@ Storage::ReturnCode Storage::Run(int max_wait) { throw util::exception("Could not open " + config.core_data_path.string() + " for reading."); } - unsigned coordinate_list_size = 0; - nodes_input_stream.read((char *)&coordinate_list_size, sizeof(unsigned)); + auto coordinate_list_size = io::readNodesSize(nodes_input_stream); shared_layout_ptr->SetBlockSize(SharedDataLayout::COORDINATE_LIST, coordinate_list_size); // we'll read a list of OSM node IDs from the same data, so set the block size for the same @@ -577,19 +573,16 @@ Storage::ReturnCode Storage::Run(int max_wait) EntryClassID *entry_class_id_ptr = shared_layout_ptr->GetBlockPtr( shared_memory_ptr, SharedDataLayout::ENTRY_CLASSID); - extractor::OriginalEdgeData current_edge_data; - for (unsigned i = 0; i < number_of_original_edges; ++i) - { - edges_input_stream.read((char *)&(current_edge_data), sizeof(extractor::OriginalEdgeData)); - via_geometry_ptr[i] = current_edge_data.via_geometry; - name_id_ptr[i] = current_edge_data.name_id; - travel_mode_ptr[i] = current_edge_data.travel_mode; - lane_data_id_ptr[i] = current_edge_data.lane_data_id; - turn_instructions_ptr[i] = current_edge_data.turn_instruction; - entry_class_id_ptr[i] = current_edge_data.entry_classid; - pre_turn_bearing_ptr[i] = current_edge_data.pre_turn_bearing; - post_turn_bearing_ptr[i] = current_edge_data.post_turn_bearing; - } + io::readEdgesData(edges_input_stream, + via_geometry_ptr, + name_id_ptr, + turn_instructions_ptr, + lane_data_id_ptr, + travel_mode_ptr, + entry_class_id_ptr, + pre_turn_bearing_ptr, + post_turn_bearing_ptr, + number_of_original_edges); edges_input_stream.close(); // load compressed geometry @@ -691,13 +684,7 @@ Storage::ReturnCode Storage::Run(int max_wait) osmnodeid_list.reset(osmnodeid_ptr, shared_layout_ptr->num_entries[SharedDataLayout::OSM_NODE_ID_LIST]); - extractor::QueryNode current_node; - for (unsigned i = 0; i < coordinate_list_size; ++i) - { - nodes_input_stream.read((char *)¤t_node, sizeof(extractor::QueryNode)); - coordinates_ptr[i] = util::Coordinate(current_node.lon, current_node.lat); - osmnodeid_list.push_back(current_node.node_id); - } + io::readNodesData(nodes_input_stream, coordinates_ptr, osmnodeid_list, coordinate_list_size); nodes_input_stream.close(); // store timestamp