From 3984dea34bcf707e440efa3d357fb4a0c330663a Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sun, 1 May 2016 13:48:56 +0200 Subject: [PATCH] Use mmap in StaticRTree --- include/util/shared_memory_vector_wrapper.hpp | 12 +++++ include/util/static_rtree.hpp | 50 ++++++++----------- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/include/util/shared_memory_vector_wrapper.hpp b/include/util/shared_memory_vector_wrapper.hpp index 74f4a8e81..9245d4c67 100644 --- a/include/util/shared_memory_vector_wrapper.hpp +++ b/include/util/shared_memory_vector_wrapper.hpp @@ -55,6 +55,12 @@ template class SharedMemoryWrapper SharedMemoryWrapper(DataT *ptr, std::size_t size) : m_ptr(ptr), m_size(size) {} + void reset(DataT *ptr, std::size_t size) + { + m_ptr = ptr; + m_size = size; + } + DataT &at(const std::size_t index) { return m_ptr[index]; } const DataT &at(const std::size_t index) const { return m_ptr[index]; } @@ -101,6 +107,12 @@ template <> class SharedMemoryWrapper return m_ptr[bucket] & (1u << offset); } + void reset(unsigned *ptr, std::size_t size) + { + m_ptr = ptr; + m_size = size; + } + std::size_t size() const { return m_size; } bool empty() const { return 0 == size(); } diff --git a/include/util/static_rtree.hpp b/include/util/static_rtree.hpp index 2bfefc591..7e4032591 100644 --- a/include/util/static_rtree.hpp +++ b/include/util/static_rtree.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -113,7 +114,10 @@ class StaticRTree uint64_t m_element_count; const std::string m_leaf_node_filename; std::shared_ptr m_coordinate_list; - boost::filesystem::ifstream leaves_stream; + + boost::iostreams::mapped_file_source m_leaves_region; + // read-only view of leaves + typename ShM::vector m_leaves; public: StaticRTree(const StaticRTree &) = delete; @@ -125,7 +129,7 @@ class StaticRTree const std::string &tree_node_filename, const std::string &leaf_node_filename, const std::vector &coordinate_list) - : m_element_count(input_data_vector.size()), m_leaf_node_filename(leaf_node_filename) + : m_element_count(input_data_vector.size()) { std::vector input_wrapper_vector(m_element_count); @@ -261,6 +265,11 @@ class StaticRTree BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty"); tree_node_file.write((char *)&size_of_tree, sizeof(std::uint32_t)); tree_node_file.write((char *)&m_search_tree[0], sizeof(TreeNode) * size_of_tree); + + m_leaves_region.open(leaf_node_filename); + std::uint64_t num_leaves = reinterpret_cast(m_leaves_region.data())[0]; + const char* m_leaves_begin = m_leaves_region.data() + sizeof(std::uint64_t); + m_leaves.reset(reinterpret_cast(m_leaves_begin), num_leaves); } explicit StaticRTree(const boost::filesystem::path &node_file, @@ -300,15 +309,17 @@ class StaticRTree throw exception("mem index file is empty"); } - leaves_stream.open(leaf_file, std::ios::binary); - leaves_stream.read((char *)&m_element_count, sizeof(uint64_t)); + m_leaves_region.open(leaf_file); + std::uint64_t num_leaves = reinterpret_cast(m_leaves_region.data())[0]; + const char* m_leaves_begin = m_leaves_region.data() + sizeof(std::uint64_t); + m_leaves.reset(reinterpret_cast(m_leaves_begin), num_leaves); } explicit StaticRTree(TreeNode *tree_node_ptr, const uint64_t number_of_nodes, const boost::filesystem::path &leaf_file, std::shared_ptr coordinate_list) - : m_search_tree(tree_node_ptr, number_of_nodes), m_leaf_node_filename(leaf_file.string()), + : m_search_tree(tree_node_ptr, number_of_nodes), m_coordinate_list(std::move(coordinate_list)) { // open leaf node file and store thread specific pointer @@ -321,8 +332,10 @@ class StaticRTree throw exception("mem index file is empty"); } - leaves_stream.open(leaf_file, std::ios::binary); - leaves_stream.read((char *)&m_element_count, sizeof(uint64_t)); + m_leaves_region.open(leaf_file); + std::uint64_t num_leaves = reinterpret_cast(m_leaves_region.data())[0]; + const char* m_leaves_begin = m_leaves_region.data() + sizeof(std::uint64_t); + m_leaves.reset(reinterpret_cast(m_leaves_begin), num_leaves); } /* Returns all features inside the bounding box. @@ -348,7 +361,7 @@ class StaticRTree if (current_tree_node.child_is_on_disk) { - LeafNode current_leaf_node = LoadLeafFromDisk(current_tree_node.children[0]); + const LeafNode& current_leaf_node = m_leaves[current_tree_node.children[0]]; for (const auto i : irange(0u, current_leaf_node.object_count)) { @@ -487,7 +500,7 @@ class StaticRTree return; } - LeafNode current_leaf_node = LoadLeafFromDisk(leaf_id); + const LeafNode& current_leaf_node = m_leaves[leaf_id]; // current object represents a block on disk for (const auto i : irange(0u, current_leaf_node.object_count)) @@ -527,25 +540,6 @@ class StaticRTree } } - inline LeafNode LoadLeafFromDisk(const std::uint32_t leaf_id) - { - LeafNode result_node; - if (!leaves_stream.is_open()) - { - leaves_stream.open(m_leaf_node_filename, std::ios::in | std::ios::binary); - } - if (!leaves_stream.good()) - { - throw exception("Could not read from leaf file."); - } - const uint64_t seek_pos = sizeof(uint64_t) + leaf_id * sizeof(LeafNode); - leaves_stream.seekg(seek_pos); - BOOST_ASSERT_MSG(leaves_stream.good(), "Seeking to position in leaf file failed."); - leaves_stream.read((char *)&result_node, sizeof(LeafNode)); - BOOST_ASSERT_MSG(leaves_stream.good(), "Reading from leaf file failed."); - return result_node; - } - template void InitializeMBRectangle(Rectangle &rectangle, const std::array &objects,