diff --git a/include/engine/datafacade/internal_datafacade.hpp b/include/engine/datafacade/internal_datafacade.hpp index 207eb9649..2ac70434b 100644 --- a/include/engine/datafacade/internal_datafacade.hpp +++ b/include/engine/datafacade/internal_datafacade.hpp @@ -14,6 +14,7 @@ #include "extractor/profile_properties.hpp" #include "extractor/query_node.hpp" #include "storage/storage_config.hpp" +#include "storage/io.hpp" #include "engine/geospatial_query.hpp" #include "util/graph_loader.hpp" #include "util/guidance/turn_lanes.hpp" @@ -150,21 +151,26 @@ class InternalDataFacade final : public BaseDataFacade void LoadGraph(const boost::filesystem::path &hsgr_path) { - util::ShM::vector node_list; - util::ShM::vector edge_list; + boost::filesystem::ifstream hsgr_input_stream(hsgr_path); + if (!hsgr_input_stream) + { + throw util::exception("Could not open " + hsgr_path.string() + " for reading."); + } - util::SimpleLogger().Write() << "loading graph from " << hsgr_path.string(); + auto header = storage::io::readHSGRHeader(hsgr_input_stream); + m_check_sum = header.checksum; - m_number_of_nodes = readHSGRFromStream(hsgr_path, node_list, edge_list, &m_check_sum); + util::ShM::vector node_list(header.number_of_nodes); + util::ShM::vector edge_list(header.number_of_edges); + + storage::io::readHSGR(hsgr_input_stream, + node_list.data(), + header.number_of_nodes, + edge_list.data(), + header.number_of_edges); - BOOST_ASSERT_MSG(0 != node_list.size(), "node list empty"); - // BOOST_ASSERT_MSG(0 != edge_list.size(), "edge list empty"); - util::SimpleLogger().Write() << "loaded " << node_list.size() << " nodes and " - << edge_list.size() << " edges"; m_query_graph = std::unique_ptr(new QueryGraph(node_list, edge_list)); - BOOST_ASSERT_MSG(0 == node_list.size(), "node list not flushed"); - BOOST_ASSERT_MSG(0 == edge_list.size(), "edge list not flushed"); util::SimpleLogger().Write() << "Data checksum is " << m_check_sum; } diff --git a/include/storage/io.hpp b/include/storage/io.hpp new file mode 100644 index 000000000..4de872a32 --- /dev/null +++ b/include/storage/io.hpp @@ -0,0 +1,68 @@ +#ifndef OSRM_STORAGE_IO_HPP_ +#define OSRM_STORAGE_IO_HPP_ + +#include "util/fingerprint.hpp" +#include "util/simple_logger.hpp" + +#include + +#include + +namespace osrm +{ +namespace storage +{ +namespace io +{ + +#pragma pack(push, 1) +struct HSGRHeader +{ + std::uint32_t checksum; + std::uint32_t number_of_nodes; + std::uint32_t number_of_edges; +}; +#pragma pack(pop) +static_assert(sizeof(HSGRHeader) == 12, "HSGRHeader is not packed"); + +// Returns the checksum and the number of nodes and number of edges +inline HSGRHeader readHSGRHeader(boost::filesystem::ifstream &input_stream) +{ + const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid(); + util::FingerPrint fingerprint_loaded; + input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(util::FingerPrint)); + if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid)) + { + util::SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build.\n" + "Reprocess to get rid of this warning."; + } + + HSGRHeader header; + input_stream.read(reinterpret_cast(&header.checksum), sizeof(header.checksum)); + input_stream.read(reinterpret_cast(&header.number_of_nodes), sizeof(header.number_of_nodes)); + input_stream.read(reinterpret_cast(&header.number_of_edges), sizeof(header.number_of_edges)); + + BOOST_ASSERT_MSG(0 != header.number_of_nodes, "number of nodes is zero"); + // number of edges can be zero, this is the case in a few test fixtures + + return header; +} + +// Needs to be called after getHSGRSize() to get the correct offset in the stream +// +template +void readHSGR(boost::filesystem::ifstream &input_stream, + NodeT *node_buffer, + std::uint32_t number_of_nodes, + EdgeT *edge_buffer, + std::uint32_t number_of_edges) +{ + input_stream.read(reinterpret_cast(node_buffer), number_of_nodes * sizeof(NodeT)); + input_stream.read(reinterpret_cast(edge_buffer), number_of_edges * sizeof(EdgeT)); +} + +} +} +} + +#endif diff --git a/include/util/fingerprint.hpp b/include/util/fingerprint.hpp index e8238c0ac..edeb52d43 100644 --- a/include/util/fingerprint.hpp +++ b/include/util/fingerprint.hpp @@ -32,6 +32,7 @@ class FingerPrint boost::uuids::uuid named_uuid; }; +static_assert(sizeof(FingerPrint) == 152, "FingerPrint has unexpected size"); static_assert(std::is_trivial::value, "FingerPrint needs to be trivial."); } } diff --git a/include/util/graph_loader.hpp b/include/util/graph_loader.hpp index 725527039..d06b66937 100644 --- a/include/util/graph_loader.hpp +++ b/include/util/graph_loader.hpp @@ -148,56 +148,6 @@ NodeID loadEdgesFromFile(std::istream &input_stream, return m; } -template -unsigned readHSGRFromStream(const boost::filesystem::path &hsgr_file, - std::vector &node_list, - std::vector &edge_list, - unsigned *check_sum) -{ - if (!boost::filesystem::exists(hsgr_file)) - { - throw exception("hsgr file does not exist"); - } - if (0 == boost::filesystem::file_size(hsgr_file)) - { - throw exception("hsgr file is empty"); - } - - boost::filesystem::ifstream hsgr_input_stream(hsgr_file, std::ios::binary); - - const FingerPrint fingerprint_valid = FingerPrint::GetValid(); - FingerPrint fingerprint_loaded; - hsgr_input_stream.read(reinterpret_cast(&fingerprint_loaded), sizeof(FingerPrint)); - if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid)) - { - SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build.\n" - "Reprocess to get rid of this warning."; - } - - unsigned number_of_nodes = 0; - unsigned number_of_edges = 0; - hsgr_input_stream.read(reinterpret_cast(check_sum), sizeof(unsigned)); - hsgr_input_stream.read(reinterpret_cast(&number_of_nodes), sizeof(unsigned)); - BOOST_ASSERT_MSG(0 != number_of_nodes, "number of nodes is zero"); - hsgr_input_stream.read(reinterpret_cast(&number_of_edges), sizeof(unsigned)); - - SimpleLogger().Write() << "number_of_nodes: " << number_of_nodes - << ", number_of_edges: " << number_of_edges; - - // BOOST_ASSERT_MSG( 0 != number_of_edges, "number of edges is zero"); - node_list.resize(number_of_nodes); - hsgr_input_stream.read(reinterpret_cast(&node_list[0]), - number_of_nodes * sizeof(NodeT)); - - edge_list.resize(number_of_edges); - if (number_of_edges > 0) - { - hsgr_input_stream.read(reinterpret_cast(&edge_list[0]), - number_of_edges * sizeof(EdgeT)); - } - - return number_of_nodes; -} } } diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 76e0ddd4f..48d3522c5 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -1,3 +1,10 @@ +#include "storage/storage.hpp" + +#include "storage/io.hpp" +#include "storage/shared_barriers.hpp" +#include "storage/shared_datatype.hpp" +#include "storage/shared_memory.hpp" + #include "contractor/query_edge.hpp" #include "extractor/compressed_edge_container.hpp" #include "extractor/guidance/turn_instruction.hpp" @@ -5,10 +12,6 @@ #include "extractor/profile_properties.hpp" #include "extractor/query_node.hpp" #include "extractor/travel_mode.hpp" -#include "storage/shared_barriers.hpp" -#include "storage/shared_datatype.hpp" -#include "storage/shared_memory.hpp" -#include "storage/storage.hpp" #include "engine/datafacade/datafacade_base.hpp" #include "util/coordinate.hpp" #include "util/exception.hpp" @@ -252,37 +255,12 @@ Storage::ReturnCode Storage::Run(int max_wait) throw util::exception("Could not open " + config.hsgr_data_path.string() + " for reading."); } - util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid(); - util::FingerPrint fingerprint_loaded; - hsgr_input_stream.read((char *)&fingerprint_loaded, sizeof(util::FingerPrint)); - if (fingerprint_loaded.TestGraphUtil(fingerprint_valid)) - { - util::SimpleLogger().Write(logDEBUG) << "Fingerprint checked out ok"; - } - else - { - util::SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build. " - "Reprocess to get rid of this warning."; - } - - // load checksum - unsigned checksum = 0; - hsgr_input_stream.read((char *)&checksum, sizeof(unsigned)); + auto hsgr_header = io::readHSGRHeader(hsgr_input_stream); shared_layout_ptr->SetBlockSize(SharedDataLayout::HSGR_CHECKSUM, 1); - // load graph node size - unsigned number_of_graph_nodes = 0; - hsgr_input_stream.read((char *)&number_of_graph_nodes, sizeof(unsigned)); - - BOOST_ASSERT_MSG((0 != number_of_graph_nodes), "number of nodes is zero"); shared_layout_ptr->SetBlockSize(SharedDataLayout::GRAPH_NODE_LIST, - number_of_graph_nodes); - - // load graph edge size - unsigned number_of_graph_edges = 0; - hsgr_input_stream.read((char *)&number_of_graph_edges, sizeof(unsigned)); - // BOOST_ASSERT_MSG(0 != number_of_graph_edges, "number of graph edges is zero"); + hsgr_header.number_of_nodes); shared_layout_ptr->SetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST, - number_of_graph_edges); + hsgr_header.number_of_edges); // load rsearch tree size boost::filesystem::ifstream tree_node_file(config.ram_index_path, std::ios::binary); @@ -477,7 +455,7 @@ Storage::ReturnCode Storage::Run(int max_wait) // hsgr checksum unsigned *checksum_ptr = shared_layout_ptr->GetBlockPtr( shared_memory_ptr, SharedDataLayout::HSGR_CHECKSUM); - *checksum_ptr = checksum; + *checksum_ptr = hsgr_header.checksum; // ram index file name char *file_index_path_ptr = shared_layout_ptr->GetBlockPtr( @@ -726,21 +704,17 @@ Storage::ReturnCode Storage::Run(int max_wait) QueryGraph::NodeArrayEntry *graph_node_list_ptr = shared_layout_ptr->GetBlockPtr( shared_memory_ptr, SharedDataLayout::GRAPH_NODE_LIST); - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST) > 0) - { - hsgr_input_stream.read((char *)graph_node_list_ptr, - shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST)); - } // load the edges of the search graph QueryGraph::EdgeArrayEntry *graph_edge_list_ptr = shared_layout_ptr->GetBlockPtr( shared_memory_ptr, SharedDataLayout::GRAPH_EDGE_LIST); - if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST) > 0) - { - hsgr_input_stream.read((char *)graph_edge_list_ptr, - shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST)); - } + + io::readHSGR(hsgr_input_stream, + graph_node_list_ptr, + hsgr_header.number_of_nodes, + graph_edge_list_ptr, + hsgr_header.number_of_edges); hsgr_input_stream.close(); // load profile properties