Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Use mmap in StaticRTree

Browse Source
This commit is contained in:
Patrick Niklaus 2016-05-01 13:48:56 +02:00
parent b11b471aa4
commit 3984dea34b
No known key found for this signature in database
GPG Key ID: E426891B5F978B1B
2 changed files with 34 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/util/shared_memory_vector_wrapper.hpp
View File

@ -55,6 +55,12 @@ template <typename DataT> class SharedMemoryWrapper
SharedMemoryWrapper(DataT *ptr, std::size_t size) : m_ptr(ptr), m_size(size) {}
void reset(DataT *ptr, std::size_t size)
{
m_ptr = ptr;
m_size = size;
}
DataT &at(const std::size_t index) { return m_ptr[index]; }
const DataT &at(const std::size_t index) const { return m_ptr[index]; }
@ -101,6 +107,12 @@ template <> class SharedMemoryWrapper<bool>
return m_ptr[bucket] & (1u << offset);
}
void reset(unsigned *ptr, std::size_t size)
{
m_ptr = ptr;
m_size = size;
}
std::size_t size() const { return m_size; }
bool empty() const { return 0 == size(); }

50
include/util/static_rtree.hpp
View File

@ -17,6 +17,7 @@
#include <boost/assert.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/iostreams/device/mapped_file.hpp>
#include <tbb/parallel_for.h>
#include <tbb/parallel_sort.h>
@ -113,7 +114,10 @@ class StaticRTree
uint64_t m_element_count;
const std::string m_leaf_node_filename;
std::shared_ptr<CoordinateListT> m_coordinate_list;
boost::filesystem::ifstream leaves_stream;
boost::iostreams::mapped_file_source m_leaves_region;
// read-only view of leaves
typename ShM<const LeafNode, true>::vector m_leaves;
public:
StaticRTree(const StaticRTree &) = delete;
@ -125,7 +129,7 @@ class StaticRTree
const std::string &tree_node_filename,
const std::string &leaf_node_filename,
const std::vector<CoordinateT> &coordinate_list)
: m_element_count(input_data_vector.size()), m_leaf_node_filename(leaf_node_filename)
: m_element_count(input_data_vector.size())
{
std::vector<WrappedInputElement> input_wrapper_vector(m_element_count);
@ -261,6 +265,11 @@ class StaticRTree
BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty");
tree_node_file.write((char *)&size_of_tree, sizeof(std::uint32_t));
tree_node_file.write((char *)&m_search_tree[0], sizeof(TreeNode) * size_of_tree);
m_leaves_region.open(leaf_node_filename);
std::uint64_t num_leaves = reinterpret_cast<const std::uint64_t*>(m_leaves_region.data())[0];
const char* m_leaves_begin = m_leaves_region.data() + sizeof(std::uint64_t);
m_leaves.reset(reinterpret_cast<const LeafNode*>(m_leaves_begin), num_leaves);
}
explicit StaticRTree(const boost::filesystem::path &node_file,
@ -300,15 +309,17 @@ class StaticRTree
throw exception("mem index file is empty");
}
leaves_stream.open(leaf_file, std::ios::binary);
leaves_stream.read((char *)&m_element_count, sizeof(uint64_t));
m_leaves_region.open(leaf_file);
std::uint64_t num_leaves = reinterpret_cast<const std::uint64_t*>(m_leaves_region.data())[0];
const char* m_leaves_begin = m_leaves_region.data() + sizeof(std::uint64_t);
m_leaves.reset(reinterpret_cast<const LeafNode*>(m_leaves_begin), num_leaves);
}
explicit StaticRTree(TreeNode *tree_node_ptr,
const uint64_t number_of_nodes,
const boost::filesystem::path &leaf_file,
std::shared_ptr<CoordinateListT> coordinate_list)
: m_search_tree(tree_node_ptr, number_of_nodes), m_leaf_node_filename(leaf_file.string()),
: m_search_tree(tree_node_ptr, number_of_nodes),
m_coordinate_list(std::move(coordinate_list))
{
// open leaf node file and store thread specific pointer
@ -321,8 +332,10 @@ class StaticRTree
throw exception("mem index file is empty");
}
leaves_stream.open(leaf_file, std::ios::binary);
leaves_stream.read((char *)&m_element_count, sizeof(uint64_t));
m_leaves_region.open(leaf_file);
std::uint64_t num_leaves = reinterpret_cast<const std::uint64_t*>(m_leaves_region.data())[0];
const char* m_leaves_begin = m_leaves_region.data() + sizeof(std::uint64_t);
m_leaves.reset(reinterpret_cast<const LeafNode*>(m_leaves_begin), num_leaves);
}
/* Returns all features inside the bounding box.
@ -348,7 +361,7 @@ class StaticRTree
if (current_tree_node.child_is_on_disk)
{
LeafNode current_leaf_node = LoadLeafFromDisk(current_tree_node.children[0]);
const LeafNode& current_leaf_node = m_leaves[current_tree_node.children[0]];
for (const auto i : irange(0u, current_leaf_node.object_count))
{
@ -487,7 +500,7 @@ class StaticRTree
return;
}
LeafNode current_leaf_node = LoadLeafFromDisk(leaf_id);
const LeafNode& current_leaf_node = m_leaves[leaf_id];
// current object represents a block on disk
for (const auto i : irange(0u, current_leaf_node.object_count))
@ -527,25 +540,6 @@ class StaticRTree
}
}
inline LeafNode LoadLeafFromDisk(const std::uint32_t leaf_id)
{
LeafNode result_node;
if (!leaves_stream.is_open())
{
leaves_stream.open(m_leaf_node_filename, std::ios::in | std::ios::binary);
}
if (!leaves_stream.good())
{
throw exception("Could not read from leaf file.");
}
const uint64_t seek_pos = sizeof(uint64_t) + leaf_id * sizeof(LeafNode);
leaves_stream.seekg(seek_pos);
BOOST_ASSERT_MSG(leaves_stream.good(), "Seeking to position in leaf file failed.");
leaves_stream.read((char *)&result_node, sizeof(LeafNode));
BOOST_ASSERT_MSG(leaves_stream.good(), "Reading from leaf file failed.");
return result_node;
}
template <typename CoordinateT>
void InitializeMBRectangle(Rectangle &rectangle,
const std::array<EdgeDataT, LEAF_NODE_SIZE> &objects,