From fe94977c9b40d75a5cf9b1cda533b287fe063844 Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Fri, 21 Oct 2016 11:25:38 -0700 Subject: [PATCH] refactor loading code of .ramIndex files and move to io.hpp --- include/storage/io.hpp | 33 +++++++++++++++++++++++++++++++-- include/util/static_rtree.hpp | 10 ++++------ src/storage/storage.cpp | 13 +++---------- 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/include/storage/io.hpp b/include/storage/io.hpp index 898b4aaeb..8d61831f4 100644 --- a/include/storage/io.hpp +++ b/include/storage/io.hpp @@ -1,6 +1,8 @@ #ifndef OSRM_STORAGE_IO_HPP_ #define OSRM_STORAGE_IO_HPP_ +#include "extractor/original_edge_data.hpp" +#include "extractor/query_node.hpp" #include "util/fingerprint.hpp" #include "util/simple_logger.hpp" @@ -68,6 +70,8 @@ void readHSGR(boost::filesystem::ifstream &input_stream, EdgeT edge_buffer[], std::uint32_t number_of_edges) { + BOOST_ASSERT(node_buffer); + BOOST_ASSERT(edge_buffer); input_stream.read(reinterpret_cast(node_buffer), number_of_nodes * sizeof(NodeT)); input_stream.read(reinterpret_cast(edge_buffer), number_of_edges * sizeof(EdgeT)); } @@ -85,6 +89,7 @@ inline void readTimestamp(boost::filesystem::ifstream ×tamp_input_stream, char timestamp[], std::size_t timestamp_length) { + BOOST_ASSERT(timestamp); timestamp_input_stream.read(timestamp, timestamp_length * sizeof(char)); } @@ -118,6 +123,12 @@ void readEdgesData(boost::filesystem::ifstream &edges_input_stream, PostTurnBearingT post_turn_bearing_list[], std::uint32_t number_of_edges) { + BOOST_ASSERT(geometry_list); + BOOST_ASSERT(name_id_list); + BOOST_ASSERT(turn_instruction_list); + BOOST_ASSERT(lane_data_id_list); + BOOST_ASSERT(travel_mode_list); + BOOST_ASSERT(entry_class_id_list); extractor::OriginalEdgeData current_edge_data; for (std::uint32_t i = 0; i < number_of_edges; ++i) { @@ -149,6 +160,7 @@ void readNodesData(boost::filesystem::ifstream &nodes_input_stream, OSMNodeIDVectorT &osmnodeid_list, std::uint32_t number_of_coordinates) { + BOOST_ASSERT(coordinate_list); extractor::QueryNode current_node; for (std::uint32_t i = 0; i < number_of_coordinates; ++i) { @@ -160,8 +172,6 @@ void readNodesData(boost::filesystem::ifstream &nodes_input_stream, } // Returns the number of indexes in a .datasource_indexes file -// TODO: The original code used uint_64t to store the number of indexes. Can someone confirm that -// this makes sense? inline std::uint64_t readDatasourceIndexesSize(boost::filesystem::ifstream &datasource_indexes_input_stream) { @@ -177,6 +187,7 @@ inline void readDatasourceIndexes(boost::filesystem::ifstream &datasource_indexe uint8_t datasource_buffer[], std::uint32_t number_of_datasource_indexes) { + BOOST_ASSERT(datasource_buffer); datasource_indexes_input_stream.read(reinterpret_cast(datasource_buffer), number_of_datasource_indexes * sizeof(std::uint8_t)); } @@ -207,6 +218,24 @@ readDatasourceNamesData(boost::filesystem::ifstream &datasource_names_input_stre } return datasource_names_data; } + +// Returns the number of ram indexes +inline std::uint32_t readRamIndexSize(boost::filesystem::ifstream &ram_index_input_stream) +{ + std::uint32_t tree_size = 0; + ram_index_input_stream.read((char *)&tree_size, sizeof(std::uint32_t)); + return tree_size; +} + +template +void readRamIndexData(boost::filesystem::ifstream &ram_index_input_stream, + RTreeNodeT rtree_buffer[], + std::uint32_t tree_size) +{ + BOOST_ASSERT(rtree_buffer); + ram_index_input_stream.read(reinterpret_cast(rtree_buffer), sizeof(RTreeNodeT) * tree_size); +} + } } } diff --git a/include/util/static_rtree.hpp b/include/util/static_rtree.hpp index 1dedffcad..7be332eb6 100644 --- a/include/util/static_rtree.hpp +++ b/include/util/static_rtree.hpp @@ -1,6 +1,7 @@ #ifndef STATIC_RTREE_HPP #define STATIC_RTREE_HPP +#include "storage/io.hpp" #include "util/bearing.hpp" #include "util/coordinate_calculation.hpp" #include "util/deallocating_vector.hpp" @@ -356,14 +357,11 @@ class StaticRTree } boost::filesystem::ifstream tree_node_file(node_file, std::ios::binary); - std::uint32_t tree_size = 0; - tree_node_file.read((char *)&tree_size, sizeof(std::uint32_t)); + + std::uint32_t tree_size = storage::io::readRamIndexSize(tree_node_file); m_search_tree.resize(tree_size); - if (tree_size > 0) - { - tree_node_file.read((char *)&m_search_tree[0], sizeof(TreeNode) * tree_size); - } + storage::io::readRamIndexData(tree_node_file, &m_search_tree[0], tree_size); MapLeafNodesFile(leaf_file); } diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index e1ae75cba..4bde35b3b 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -266,8 +266,7 @@ Storage::ReturnCode Storage::Run(int max_wait) // load rsearch tree size boost::filesystem::ifstream tree_node_file(config.ram_index_path, std::ios::binary); - uint32_t tree_size = 0; - tree_node_file.read((char *)&tree_size, sizeof(uint32_t)); + std::uint32_t tree_size = io::readRamIndexSize(tree_node_file); shared_layout_ptr->SetBlockSize(SharedDataLayout::R_SEARCH_TREE, tree_size); // allocate space in shared memory for profile properties @@ -677,7 +676,6 @@ Storage::ReturnCode Storage::Run(int max_wait) util::PackedVector osmnodeid_list; osmnodeid_list.reset(osmnodeid_ptr, shared_layout_ptr->num_entries[SharedDataLayout::OSM_NODE_ID_LIST]); - io::readNodesData(nodes_input_stream, coordinates_ptr, osmnodeid_list, coordinate_list_size); nodes_input_stream.close(); @@ -687,14 +685,9 @@ Storage::ReturnCode Storage::Run(int max_wait) io::readTimestamp(timestamp_stream, timestamp_ptr, timestamp_size); // store search tree portion of rtree - char *rtree_ptr = shared_layout_ptr->GetBlockPtr(shared_memory_ptr, + RTreeNode *rtree_ptrtest = shared_layout_ptr->GetBlockPtr(shared_memory_ptr, SharedDataLayout::R_SEARCH_TREE); - - if (tree_size > 0) - { - tree_node_file.read(rtree_ptr, sizeof(RTreeNode) * tree_size); - } - tree_node_file.close(); + io::readRamIndexData(tree_node_file, rtree_ptrtest, tree_size); // load core markers std::vector unpacked_core_markers(number_of_core_markers);