Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Port .ramIndex to tar file and mmap .fileIndex directly

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-21 17:09:59 +00:00
parent 86ffce3a50
commit 8152dcfb4c
10 changed files with 205 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View File

@ -254,17 +254,21 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
"Is any data loaded into shared memory?" + SOURCE_REF); "Is any data loaded into shared memory?" + SOURCE_REF);
} }
auto tree_nodes_ptr = const auto rtree_ptr =
data_layout.GetBlockPtr<RTreeNode>(memory_block, storage::DataLayout::R_SEARCH_TREE); data_layout.GetBlockPtr<RTreeNode>(memory_block, storage::DataLayout::R_SEARCH_TREE);
auto tree_level_sizes_ptr = data_layout.GetBlockPtr<std::uint64_t>( util::vector_view<RTreeNode> search_tree(
memory_block, storage::DataLayout::R_SEARCH_TREE_LEVELS); rtree_ptr, data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE));
m_static_rtree.reset(
new SharedRTree(tree_nodes_ptr, const auto rtree_levelstarts_ptr = data_layout.GetBlockPtr<std::uint64_t>(
data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE), memory_block, storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS);
tree_level_sizes_ptr, util::vector_view<std::uint64_t> rtree_level_starts(
data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVELS), rtree_levelstarts_ptr,
data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS));
m_static_rtree.reset(new SharedRTree{std::move(search_tree),
std::move(rtree_level_starts),
file_index_path, file_index_path,
m_coordinate_list)); m_coordinate_list});
m_geospatial_query.reset( m_geospatial_query.reset(
new SharedGeospatialQuery(*m_static_rtree, m_coordinate_list, *this)); new SharedGeospatialQuery(*m_static_rtree, m_coordinate_list, *this));
} }

21
include/extractor/files.hpp
View File

@ -450,13 +450,32 @@ void readEdgeBasedNodeWeights(const boost::filesystem::path &path, NodeWeigtsVec
} }
template <typename NodeWeigtsVectorT> template <typename NodeWeigtsVectorT>
void writeEdgeBasedNodeWeights(const boost::filesystem::path &path, const NodeWeigtsVectorT &weights) void writeEdgeBasedNodeWeights(const boost::filesystem::path &path,
const NodeWeigtsVectorT &weights)
{ {
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint}; storage::tar::FileWriter writer{path, fingerprint};
storage::serialization::write(writer, "/extractor/edge_based_node_weights", weights); storage::serialization::write(writer, "/extractor/edge_based_node_weights", weights);
} }
template <typename RTreeT>
void writeRamIndex(const boost::filesystem::path &path, const RTreeT &rtree)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
util::serialization::write(writer, "/common/rtree", rtree);
}
template <typename RTreeT>
void readRamIndex(const boost::filesystem::path &path, RTreeT &rtree)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
util::serialization::read(reader, "/common/rtree", rtree);
}
} }
} }
} }

4
include/storage/shared_datatype.hpp
View File

@ -40,7 +40,7 @@ const constexpr char *block_id_to_name[] = {"IGNORE_BLOCK",
"TURN_INSTRUCTION", "TURN_INSTRUCTION",
"ENTRY_CLASSID", "ENTRY_CLASSID",
"R_SEARCH_TREE", "R_SEARCH_TREE",
"R_SEARCH_TREE_LEVELS", "R_SEARCH_TREE_LEVEL_STARTS",
"GEOMETRIES_INDEX", "GEOMETRIES_INDEX",
"GEOMETRIES_NODE_LIST", "GEOMETRIES_NODE_LIST",
"GEOMETRIES_FWD_WEIGHT_LIST", "GEOMETRIES_FWD_WEIGHT_LIST",
@ -120,7 +120,7 @@ class DataLayout
TURN_INSTRUCTION, TURN_INSTRUCTION,
ENTRY_CLASSID, ENTRY_CLASSID,
R_SEARCH_TREE, R_SEARCH_TREE,
R_SEARCH_TREE_LEVELS, R_SEARCH_TREE_LEVEL_STARTS,
GEOMETRIES_INDEX, GEOMETRIES_INDEX,
GEOMETRIES_NODE_LIST, GEOMETRIES_NODE_LIST,
GEOMETRIES_FWD_WEIGHT_LIST, GEOMETRIES_FWD_WEIGHT_LIST,

21
include/util/serialization.hpp
View File

@ -6,6 +6,7 @@
#include "util/range_table.hpp" #include "util/range_table.hpp"
#include "util/static_graph.hpp" #include "util/static_graph.hpp"
#include "util/indexed_data.hpp" #include "util/indexed_data.hpp"
#include "util/static_rtree.hpp"
#include "storage/io.hpp" #include "storage/io.hpp"
#include "storage/serialization.hpp" #include "storage/serialization.hpp"
@ -185,6 +186,26 @@ inline void write(storage::tar::FileWriter &writer,
storage::serialization::write(writer, name + "/blocks", index_data.blocks); storage::serialization::write(writer, name + "/blocks", index_data.blocks);
storage::serialization::write(writer, name + "/values", index_data.values); storage::serialization::write(writer, name + "/values", index_data.values);
} }
template <class EdgeDataT,
storage::Ownership Ownership,
std::uint32_t BRANCHING_FACTOR,
std::uint32_t LEAF_PAGE_SIZE>
void read(storage::tar::FileReader &reader, const std::string& name, util::StaticRTree<EdgeDataT, Ownership, BRANCHING_FACTOR, LEAF_PAGE_SIZE> &rtree)
{
storage::serialization::read(reader, name + "/search_tree", rtree.m_search_tree);
storage::serialization::read(reader, name + "/search_tree_level_starts", rtree.m_tree_level_starts);
}
template <class EdgeDataT,
storage::Ownership Ownership,
std::uint32_t BRANCHING_FACTOR,
std::uint32_t LEAF_PAGE_SIZE>
void write(storage::tar::FileWriter &writer, const std::string& name, const util::StaticRTree<EdgeDataT, Ownership, BRANCHING_FACTOR, LEAF_PAGE_SIZE> &rtree){
storage::serialization::write(writer, name + "/search_tree", rtree.m_search_tree);
storage::serialization::write(writer, name + "/search_tree_level_starts", rtree.m_tree_level_starts);
}
} }
} }
} }

207
include/util/static_rtree.hpp
View File

@ -1,7 +1,8 @@
#ifndef STATIC_RTREE_HPP #ifndef STATIC_RTREE_HPP
#define STATIC_RTREE_HPP #define STATIC_RTREE_HPP
#include "storage/io.hpp" #include "storage/tar_fwd.hpp"
#include "util/bearing.hpp" #include "util/bearing.hpp"
#include "util/coordinate_calculation.hpp" #include "util/coordinate_calculation.hpp"
#include "util/deallocating_vector.hpp" #include "util/deallocating_vector.hpp"
@ -35,20 +36,35 @@
#include <string> #include <string>
#include <vector> #include <vector>
// An extended alignment is implementation-defined, so use compiler attributes
// until alignas(LEAF_PAGE_SIZE) is compiler-independent.
#if defined(_MSC_VER)
#define ALIGNED(x) __declspec(align(x))
#elif defined(__GNUC__)
#define ALIGNED(x) __attribute__((aligned(x)))
#else
#define ALIGNED(x)
#endif
namespace osrm namespace osrm
{ {
namespace util namespace util
{ {
template <class EdgeDataT,
storage::Ownership Ownership = storage::Ownership::Container,
std::uint32_t BRANCHING_FACTOR = 64,
std::uint32_t LEAF_PAGE_SIZE = 4096>
class StaticRTree;
namespace serialization
{
template <class EdgeDataT,
storage::Ownership Ownership,
std::uint32_t BRANCHING_FACTOR,
std::uint32_t LEAF_PAGE_SIZE>
inline void read(storage::tar::FileReader &reader,
const std::string &name,
util::StaticRTree<EdgeDataT, Ownership, BRANCHING_FACTOR, LEAF_PAGE_SIZE> &rtree);
template <class EdgeDataT,
storage::Ownership Ownership,
std::uint32_t BRANCHING_FACTOR,
std::uint32_t LEAF_PAGE_SIZE>
inline void
write(storage::tar::FileWriter &writer,
const std::string &name,
const util::StaticRTree<EdgeDataT, Ownership, BRANCHING_FACTOR, LEAF_PAGE_SIZE> &rtree);
}
/*** /***
* Static RTree for serving nearest neighbour queries * Static RTree for serving nearest neighbour queries
@ -57,9 +73,9 @@ namespace util
*/ */
template <class EdgeDataT, template <class EdgeDataT,
storage::Ownership Ownership = storage::Ownership::Container, storage::Ownership Ownership,
std::uint32_t BRANCHING_FACTOR = 64, std::uint32_t BRANCHING_FACTOR,
std::uint32_t LEAF_PAGE_SIZE = 4096> std::uint32_t LEAF_PAGE_SIZE>
class StaticRTree class StaticRTree
{ {
/********************************************************** /**********************************************************
@ -236,39 +252,31 @@ class StaticRTree
std::uint32_t segment_index; std::uint32_t segment_index;
}; };
// We use a const view type when we don't own the data, otherwise // Representation of the in-memory search tree
// we use a mutable type (usually becase we're building the tree) Vector<TreeNode> m_search_tree;
using TreeViewType = typename std::conditional<Ownership == storage::Ownership::View,
const Vector<const TreeNode>,
Vector<TreeNode>>::type;
TreeViewType m_search_tree;
// Reference to the actual lon/lat data we need for doing math // Reference to the actual lon/lat data we need for doing math
const Vector<Coordinate> &m_coordinate_list; const Vector<Coordinate> &m_coordinate_list;
// Holds the number of TreeNodes in each level.
// We always start with the root node, so
// m_tree_level_sizes[0] should always be 1
std::vector<std::uint64_t> m_tree_level_sizes;
// Holds the start indexes of each level in m_search_tree // Holds the start indexes of each level in m_search_tree
std::vector<std::uint64_t> m_tree_level_starts; Vector<std::uint64_t> m_tree_level_starts;
// mmap'd .fileIndex file // mmap'd .fileIndex file
boost::iostreams::mapped_file_source m_objects_region; boost::iostreams::mapped_file m_objects_region;
// This is a view of the EdgeDataT data mmap'd from the .fileIndex file // This is a view of the EdgeDataT data mmap'd from the .fileIndex file
util::vector_view<const EdgeDataT> m_objects; util::vector_view<EdgeDataT> m_objects;
public: public:
StaticRTree(const StaticRTree &) = delete; StaticRTree(const StaticRTree &) = delete;
StaticRTree &operator=(const StaticRTree &) = delete; StaticRTree &operator=(const StaticRTree &) = delete;
StaticRTree(StaticRTree &&) = default;
StaticRTree &operator=(StaticRTree &&) = default;
// Construct a packed Hilbert-R-Tree with Kamel-Faloutsos algorithm [1] // Construct a packed Hilbert-R-Tree with Kamel-Faloutsos algorithm [1]
explicit StaticRTree(const std::vector<EdgeDataT> &input_data_vector, explicit StaticRTree(const std::vector<EdgeDataT> &input_data_vector,
const std::string &tree_node_filename, const Vector<Coordinate> &coordinate_list,
const std::string &leaf_node_filename, const boost::filesystem::path &on_disk_file_name)
const Vector<Coordinate> &coordinate_list) : m_coordinate_list(coordinate_list), m_objects{mmapFile<EdgeDataT>(
: m_coordinate_list(coordinate_list) on_disk_file_name,
m_objects_region,
input_data_vector.size() * sizeof(EdgeDataT))}
{ {
const auto element_count = input_data_vector.size(); const auto element_count = input_data_vector.size();
std::vector<WrappedInputElement> input_wrapper_vector(element_count); std::vector<WrappedInputElement> input_wrapper_vector(element_count);
@ -304,8 +312,7 @@ class StaticRTree
// sort the hilbert-value representatives // sort the hilbert-value representatives
tbb::parallel_sort(input_wrapper_vector.begin(), input_wrapper_vector.end()); tbb::parallel_sort(input_wrapper_vector.begin(), input_wrapper_vector.end());
{ {
storage::io::FileWriter leaf_node_file(leaf_node_filename,
storage::io::FileWriter::HasNoFingerprint);
// Note, we can't just write everything in one go, because the input_data_vector // Note, we can't just write everything in one go, because the input_data_vector
// is not sorted by hilbert code, only the input_wrapper_vector is in the correct // is not sorted by hilbert code, only the input_wrapper_vector is in the correct
// order. Instead, we iterate over input_wrapper_vector, copy the hilbert-indexed // order. Instead, we iterate over input_wrapper_vector, copy the hilbert-indexed
@ -315,13 +322,12 @@ class StaticRTree
// Create the first level of TreeNodes - each bounding LEAF_NODE_COUNT EdgeDataT // Create the first level of TreeNodes - each bounding LEAF_NODE_COUNT EdgeDataT
// objects. // objects.
std::size_t wrapped_element_index = 0; std::size_t wrapped_element_index = 0;
auto objects_iter = m_objects.begin();
while (wrapped_element_index < element_count) while (wrapped_element_index < element_count)
{ {
TreeNode current_node; TreeNode current_node;
std::array<EdgeDataT, LEAF_NODE_SIZE> objects;
std::uint32_t object_count = 0;
// Loop over the next block of EdgeDataT, calculate the bounding box // Loop over the next block of EdgeDataT, calculate the bounding box
// for the block, and save the data to write to disk in the correct // for the block, and save the data to write to disk in the correct
// order. // order.
@ -333,8 +339,7 @@ class StaticRTree
input_wrapper_vector[wrapped_element_index].m_original_index; input_wrapper_vector[wrapped_element_index].m_original_index;
const EdgeDataT &object = input_data_vector[input_object_index]; const EdgeDataT &object = input_data_vector[input_object_index];
object_count += 1; *objects_iter++ = object;
objects[object_index] = object;
Coordinate projected_u{ Coordinate projected_u{
web_mercator::fromWGS84(Coordinate{m_coordinate_list[object.u]})}; web_mercator::fromWGS84(Coordinate{m_coordinate_list[object.u]})};
@ -361,19 +366,19 @@ class StaticRTree
current_node.minimum_bounding_rectangle.MergeBoundingBoxes(rectangle); current_node.minimum_bounding_rectangle.MergeBoundingBoxes(rectangle);
} }
// Write out our EdgeDataT block to the leaf node file
leaf_node_file.WriteFrom(objects.data(), object_count);
m_search_tree.emplace_back(current_node); m_search_tree.emplace_back(current_node);
} }
// leaf_node_file wil be RAII closed at this point
} }
// Should hold the number of nodes at the lowest level of the graph (closest // Should hold the number of nodes at the lowest level of the graph (closest
// to the data) // to the data)
std::uint32_t nodes_in_previous_level = m_search_tree.size(); std::uint32_t nodes_in_previous_level = m_search_tree.size();
m_tree_level_sizes.push_back(nodes_in_previous_level);
// Holds the number of TreeNodes in each level.
// We always start with the root node, so
// m_tree_level_sizes[0] should always be 1
std::vector<std::uint64_t> tree_level_sizes;
tree_level_sizes.push_back(nodes_in_previous_level);
// Now, repeatedly create levels of nodes that contain BRANCHING_FACTOR // Now, repeatedly create levels of nodes that contain BRANCHING_FACTOR
// nodes from the previous level. // nodes from the previous level.
@ -408,7 +413,7 @@ class StaticRTree
m_search_tree.emplace_back(parent_node); m_search_tree.emplace_back(parent_node);
} }
nodes_in_previous_level = nodes_in_current_level; nodes_in_previous_level = nodes_in_current_level;
m_tree_level_sizes.push_back(nodes_in_previous_level); tree_level_sizes.push_back(nodes_in_previous_level);
} }
// At this point, we've got our tree built, but the nodes are in a weird order. // At this point, we've got our tree built, but the nodes are in a weird order.
// Next thing we'll do is flip it around so that we don't end up with a lot of // Next thing we'll do is flip it around so that we don't end up with a lot of
@ -419,14 +424,15 @@ class StaticRTree
std::reverse(m_search_tree.begin(), m_search_tree.end()); std::reverse(m_search_tree.begin(), m_search_tree.end());
// Same for the level sizes - root node / base level is at 0 // Same for the level sizes - root node / base level is at 0
std::reverse(m_tree_level_sizes.begin(), m_tree_level_sizes.end()); std::reverse(tree_level_sizes.begin(), tree_level_sizes.end());
// The first level starts at 0 // The first level starts at 0
m_tree_level_starts = {0}; m_tree_level_starts = {0};
// The remaining levels start at the partial sum of the preceeding level sizes // The remaining levels start at the partial sum of the preceeding level sizes
std::partial_sum(m_tree_level_sizes.begin(), std::partial_sum(tree_level_sizes.begin(),
m_tree_level_sizes.end() - 1, tree_level_sizes.end(),
std::back_inserter(m_tree_level_starts)); std::back_inserter(m_tree_level_starts));
BOOST_ASSERT(m_tree_level_starts.size() >= 2);
// Now we have to flip the coordinates within each level so that math is easier // Now we have to flip the coordinates within each level so that math is easier
// later on. The workflow here is: // later on. The workflow here is:
@ -438,58 +444,22 @@ class StaticRTree
// 0 12 345 6789 // 0 12 345 6789
// This ordering keeps the position math easy to understand during later // This ordering keeps the position math easy to understand during later
// searches // searches
for (auto i : irange<std::size_t>(0, m_tree_level_sizes.size())) for (auto i : irange<std::size_t>(0, tree_level_sizes.size()))
{ {
std::reverse(m_search_tree.begin() + m_tree_level_starts[i], std::reverse(m_search_tree.begin() + m_tree_level_starts[i],
m_search_tree.begin() + m_tree_level_starts[i] + m_tree_level_sizes[i]); m_search_tree.begin() + m_tree_level_starts[i] + tree_level_sizes[i]);
} }
// Write all the TreeNode data to disk
{
storage::io::FileWriter tree_node_file(tree_node_filename,
storage::io::FileWriter::GenerateFingerprint);
std::uint64_t size_of_tree = m_search_tree.size();
BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty");
tree_node_file.WriteFrom(size_of_tree);
tree_node_file.WriteFrom(m_search_tree);
tree_node_file.WriteFrom(static_cast<std::uint64_t>(m_tree_level_sizes.size()));
tree_node_file.WriteFrom(m_tree_level_sizes);
}
m_objects = mmapFile<EdgeDataT>(leaf_node_filename, m_objects_region);
} }
/** /**
* Constructs an r-tree from already prepared files on disk (generated by the previous * Constructs an empty RTree for de-serialization.
* constructor)
*/ */
explicit StaticRTree(const boost::filesystem::path &node_file, template <typename = std::enable_if<Ownership == storage::Ownership::Container>>
const boost::filesystem::path &leaf_file, explicit StaticRTree(const boost::filesystem::path &on_disk_file_name,
const Vector<Coordinate> &coordinate_list) const Vector<Coordinate> &coordinate_list)
: m_coordinate_list(coordinate_list) : m_coordinate_list(coordinate_list)
{ {
storage::io::FileReader tree_node_file(node_file, m_objects = mmapFile<EdgeDataT>(on_disk_file_name, m_objects_region);
storage::io::FileReader::VerifyFingerprint);
const auto tree_size = tree_node_file.ReadElementCount64();
m_search_tree.resize(tree_size);
tree_node_file.ReadInto(m_search_tree);
const auto levels_array_size = tree_node_file.ReadElementCount64();
m_tree_level_sizes.resize(levels_array_size);
tree_node_file.ReadInto(m_tree_level_sizes);
// The first level always starts at 0
m_tree_level_starts = {0};
// The remaining levels start at the partial sum of the preceeding level sizes
std::partial_sum(m_tree_level_sizes.begin(),
m_tree_level_sizes.end() - 1,
std::back_inserter(m_tree_level_starts));
m_objects = mmapFile<EdgeDataT>(leaf_file, m_objects_region);
} }
/** /**
@ -498,22 +468,15 @@ class StaticRTree
* These memory blocks basically just contain the files read into RAM, * These memory blocks basically just contain the files read into RAM,
* excep the .fileIndex file always stays on disk, and we mmap() it as usual * excep the .fileIndex file always stays on disk, and we mmap() it as usual
*/ */
explicit StaticRTree(const TreeNode *tree_node_ptr, explicit StaticRTree(Vector<TreeNode> search_tree_,
const uint64_t number_of_nodes, Vector<std::uint64_t> tree_level_starts,
const std::uint64_t *level_sizes_ptr, const boost::filesystem::path &on_disk_file_name,
const std::size_t number_of_levels,
const boost::filesystem::path &leaf_file,
const Vector<Coordinate> &coordinate_list) const Vector<Coordinate> &coordinate_list)
: m_search_tree(tree_node_ptr, number_of_nodes), m_coordinate_list(coordinate_list), : m_search_tree(std::move(search_tree_)), m_coordinate_list(coordinate_list),
m_tree_level_sizes(level_sizes_ptr, level_sizes_ptr + number_of_levels) m_tree_level_starts(std::move(tree_level_starts))
{ {
// The first level starts at 0 BOOST_ASSERT(m_tree_level_starts.size() >= 2);
m_tree_level_starts = {0}; m_objects = mmapFile<EdgeDataT>(on_disk_file_name, m_objects_region);
// The remaining levels start at the partial sum of the preceeding level sizes
std::partial_sum(m_tree_level_sizes.begin(),
m_tree_level_sizes.end() - 1,
std::back_inserter(m_tree_level_starts));
m_objects = mmapFile<EdgeDataT>(leaf_file, m_objects_region);
} }
/* Returns all features inside the bounding box. /* Returns all features inside the bounding box.
@ -734,6 +697,13 @@ class StaticRTree
} }
} }
std::uint64_t GetLevelSize(const std::size_t level) const
{
BOOST_ASSERT(m_tree_level_starts.size() > level + 1);
BOOST_ASSERT(m_tree_level_starts[level + 1] >= m_tree_level_starts[level]);
return m_tree_level_starts[level + 1] - m_tree_level_starts[level];
}
/** /**
* Calculates the absolute position of child data in our packed data * Calculates the absolute position of child data in our packed data
* vectors. * vectors.
@ -769,22 +739,31 @@ class StaticRTree
const std::uint64_t first_child_index = const std::uint64_t first_child_index =
m_tree_level_starts[parent.level + 1] + parent.offset * BRANCHING_FACTOR; m_tree_level_starts[parent.level + 1] + parent.offset * BRANCHING_FACTOR;
const std::uint64_t end_child_index = std::min( const std::uint64_t end_child_index =
first_child_index + BRANCHING_FACTOR, std::min(first_child_index + BRANCHING_FACTOR,
m_tree_level_starts[parent.level + 1] + m_tree_level_sizes[parent.level + 1]); m_tree_level_starts[parent.level + 1] + GetLevelSize(parent.level + 1));
BOOST_ASSERT(first_child_index < std::numeric_limits<std::uint32_t>::max()); BOOST_ASSERT(first_child_index < std::numeric_limits<std::uint32_t>::max());
BOOST_ASSERT(end_child_index < std::numeric_limits<std::uint32_t>::max()); BOOST_ASSERT(end_child_index < std::numeric_limits<std::uint32_t>::max());
BOOST_ASSERT(end_child_index <= m_search_tree.size()); BOOST_ASSERT(end_child_index <= m_search_tree.size());
BOOST_ASSERT(end_child_index <= m_tree_level_starts[parent.level + 1] + BOOST_ASSERT(end_child_index <=
m_tree_level_sizes[parent.level + 1]); m_tree_level_starts[parent.level + 1] + GetLevelSize(parent.level + 1));
return irange<std::size_t>(first_child_index, end_child_index); return irange<std::size_t>(first_child_index, end_child_index);
} }
} }
bool is_leaf(const TreeIndex &treeindex) const bool is_leaf(const TreeIndex &treeindex) const
{ {
return treeindex.level == m_tree_level_starts.size() - 1; BOOST_ASSERT(m_tree_level_starts.size() >= 2);
return treeindex.level == m_tree_level_starts.size() - 2;
} }
friend void serialization::read<EdgeDataT, Ownership, BRANCHING_FACTOR, LEAF_PAGE_SIZE>(
storage::tar::FileReader &reader,
const std::string &name,
StaticRTree &rtree);
friend void serialization::write<EdgeDataT, Ownership, BRANCHING_FACTOR, LEAF_PAGE_SIZE>(
storage::tar::FileWriter &writer, const std::string &name, const StaticRTree &rtree);
}; };
//[1] "On Packing R-Trees"; I. Kamel, C. Faloutsos; 1993; DOI: 10.1145/170088.170403 //[1] "On Packing R-Trees"; I. Kamel, C. Faloutsos; 1993; DOI: 10.1145/170088.170403

2
src/benchmarks/CMakeLists.txt
View File

@ -6,7 +6,7 @@ file(GLOB PackedVectorBenchmarkSources packed_vector.cpp)
add_executable(rtree-bench add_executable(rtree-bench
EXCLUDE_FROM_ALL EXCLUDE_FROM_ALL
${RTreeBenchmarkSources} ${RTreeBenchmarkSources}
$<TARGET_OBJECTS:UTIL>) $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
target_include_directories(rtree-bench target_include_directories(rtree-bench
PUBLIC PUBLIC

13
src/benchmarks/static_rtree.cpp
View File

@ -1,6 +1,9 @@
#include "util/static_rtree.hpp" #include "util/static_rtree.hpp"
#include "extractor/edge_based_node_segment.hpp" #include "extractor/edge_based_node_segment.hpp"
#include "extractor/query_node.hpp" #include "extractor/query_node.hpp"
#include "extractor/files.hpp"
#include "extractor/packed_osm_ids.hpp"
#include "mocks/mock_datafacade.hpp" #include "mocks/mock_datafacade.hpp"
#include "storage/io.hpp" #include "storage/io.hpp"
#include "engine/geospatial_query.hpp" #include "engine/geospatial_query.hpp"
@ -32,11 +35,10 @@ using BenchStaticRTree = util::StaticRTree<RTreeLeaf, storage::Ownership::Contai
std::vector<util::Coordinate> loadCoordinates(const boost::filesystem::path &nodes_file) std::vector<util::Coordinate> loadCoordinates(const boost::filesystem::path &nodes_file)
{ {
storage::io::FileReader nodes_path_file_reader(nodes_file,
storage::io::FileReader::VerifyFingerprint);
std::vector<util::Coordinate> coords; std::vector<util::Coordinate> coords;
storage::serialization::read(nodes_path_file_reader, coords); extractor::PackedOSMIDs nodes;
extractor::files::readNodes(nodes_file, coords, nodes);
return coords; return coords;
} }
@ -99,7 +101,8 @@ int main(int argc, char **argv)
auto coords = osrm::benchmarks::loadCoordinates(nodes_path); auto coords = osrm::benchmarks::loadCoordinates(nodes_path);
osrm::benchmarks::BenchStaticRTree rtree(ram_path, file_path, coords); osrm::benchmarks::BenchStaticRTree rtree(file_path, coords);
osrm::extractor::files::readRamIndex(ram_path, rtree);
osrm::benchmarks::benchmark(rtree, 10000); osrm::benchmarks::benchmark(rtree, 10000);

8
src/extractor/extractor.cpp
View File

@ -805,10 +805,10 @@ void Extractor::BuildRTree(std::vector<EdgeBasedNodeSegment> edge_based_node_seg
edge_based_node_segments.resize(new_size); edge_based_node_segments.resize(new_size);
TIMER_START(construction); TIMER_START(construction);
util::StaticRTree<EdgeBasedNodeSegment> rtree(edge_based_node_segments, util::StaticRTree<EdgeBasedNodeSegment> rtree(edge_based_node_segments, coordinates,
config.GetPath(".osrm.ramIndex").string(), config.GetPath(".osrm.fileIndex"));
config.GetPath(".osrm.fileIndex").string(),
coordinates); files::writeRamIndex(config.GetPath(".osrm.ramIndex"), rtree);
TIMER_STOP(construction); TIMER_STOP(construction);
util::Log() << "finished r-tree construction in " << TIMER_SEC(construction) << " seconds"; util::Log() << "finished r-tree construction in " << TIMER_SEC(construction) << " seconds";

43
src/storage/storage.cpp
View File

@ -244,19 +244,6 @@ void Storage::PopulateLayout(DataLayout &layout)
make_block<char>(absolute_file_index_path.string().length() + 1)); make_block<char>(absolute_file_index_path.string().length() + 1));
} }
// load rsearch tree size
{
io::FileReader tree_node_file(config.GetPath(".osrm.ramIndex"),
io::FileReader::VerifyFingerprint);
const auto tree_size = tree_node_file.ReadElementCount64();
layout.SetBlock(DataLayout::R_SEARCH_TREE, make_block<RTreeNode>(tree_size));
tree_node_file.Skip<RTreeNode>(tree_size);
const auto tree_levels_size = tree_node_file.ReadElementCount64();
layout.SetBlock(DataLayout::R_SEARCH_TREE_LEVELS,
make_block<std::uint64_t>(tree_levels_size));
}
std::unordered_map<std::string, DataLayout::BlockID> name_to_block_id = { std::unordered_map<std::string, DataLayout::BlockID> name_to_block_id = {
{"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST}, {"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST},
{"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST}, {"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST},
@ -333,6 +320,8 @@ void Storage::PopulateLayout(DataLayout &layout)
{"/common/turn_data/connectivity_checksum", DataLayout::IGNORE_BLOCK}, {"/common/turn_data/connectivity_checksum", DataLayout::IGNORE_BLOCK},
{"/common/names/blocks", DataLayout::NAME_BLOCKS}, {"/common/names/blocks", DataLayout::NAME_BLOCKS},
{"/common/names/values", DataLayout::NAME_VALUES}, {"/common/names/values", DataLayout::NAME_VALUES},
{"/common/rtree/search_tree", DataLayout::R_SEARCH_TREE},
{"/common/rtree/search_tree_level_starts", DataLayout::R_SEARCH_TREE_LEVEL_STARTS},
}; };
std::vector<NamedBlock> blocks; std::vector<NamedBlock> blocks;
@ -357,6 +346,7 @@ void Storage::PopulateLayout(DataLayout &layout)
{REQUIRED, config.GetPath(".osrm.turn_duration_penalties")}, {REQUIRED, config.GetPath(".osrm.turn_duration_penalties")},
{REQUIRED, config.GetPath(".osrm.edges")}, {REQUIRED, config.GetPath(".osrm.edges")},
{REQUIRED, config.GetPath(".osrm.names")}, {REQUIRED, config.GetPath(".osrm.names")},
{REQUIRED, config.GetPath(".osrm.ramIndex")},
}; };
for (const auto &file : tar_files) for (const auto &file : tar_files)
@ -636,22 +626,27 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// store search tree portion of rtree // store search tree portion of rtree
{ {
io::FileReader tree_node_file(config.GetPath(".osrm.ramIndex"),
io::FileReader::VerifyFingerprint);
// perform this read so that we're at the right stream position for the next
// read.
tree_node_file.Skip<std::uint64_t>(1);
const auto rtree_ptr = const auto rtree_ptr =
layout.GetBlockPtr<RTreeNode, true>(memory_ptr, DataLayout::R_SEARCH_TREE); layout.GetBlockPtr<RTreeNode, true>(memory_ptr, DataLayout::R_SEARCH_TREE);
util::vector_view<RTreeNode> search_tree(
rtree_ptr, layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE));
tree_node_file.ReadInto(rtree_ptr, layout.GetBlockEntries(DataLayout::R_SEARCH_TREE)); const auto rtree_levelstarts_ptr = layout.GetBlockPtr<std::uint64_t, true>(
memory_ptr, DataLayout::R_SEARCH_TREE_LEVEL_STARTS);
util::vector_view<std::uint64_t> rtree_level_starts(
rtree_levelstarts_ptr,
layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS));
tree_node_file.Skip<std::uint64_t>(1); // we need this purely for the interface
const auto rtree_levelsizes_ptr = util::vector_view<util::Coordinate> empty_coords;
layout.GetBlockPtr<std::uint64_t, true>(memory_ptr, DataLayout::R_SEARCH_TREE_LEVELS);
tree_node_file.ReadInto(rtree_levelsizes_ptr, util::StaticRTree<RTreeLeaf, storage::Ownership::View> rtree{
layout.GetBlockEntries(DataLayout::R_SEARCH_TREE_LEVELS)); std::move(search_tree),
std::move(rtree_level_starts),
config.GetPath(".osrm.fileIndex"),
empty_coords};
extractor::files::readRamIndex(config.GetPath(".osrm.ramIndex"), rtree);
} }
// load profile properties // load profile properties

61
unit_tests/util/static_rtree.cpp
View File

@ -8,6 +8,7 @@
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
#include "mocks/mock_datafacade.hpp" #include "mocks/mock_datafacade.hpp"
#include "../common/temporary_file.hpp"
#include <boost/functional/hash.hpp> #include <boost/functional/hash.hpp>
#include <boost/test/auto_unit_test.hpp> #include <boost/test/auto_unit_test.hpp>
@ -244,60 +245,53 @@ void sampling_verify_rtree(RTreeT &rtree,
} }
} }
template <typename FixtureT, typename RTreeT = TestStaticRTree> template <typename RTreeT, typename FixtureT>
void build_rtree(const std::string &prefix, auto make_rtree(const boost::filesystem::path &path, FixtureT &fixture)
FixtureT *fixture,
std::string &leaves_path,
std::string &nodes_path)
{ {
nodes_path = prefix + ".ramIndex"; return RTreeT(fixture.edges, fixture.coords, path);
leaves_path = prefix + ".fileIndex";
RTreeT r(fixture->edges, nodes_path, leaves_path, fixture->coords);
} }
template <typename RTreeT = TestStaticRTree, typename FixtureT> template <typename RTreeT = TestStaticRTree, typename FixtureT>
void construction_test(const std::string &prefix, FixtureT *fixture) void construction_test(const std::string &path, FixtureT &fixture)
{ {
std::string leaves_path; std::string leaves_path;
std::string nodes_path; std::string nodes_path;
build_rtree<FixtureT, RTreeT>(prefix, fixture, leaves_path, nodes_path); auto rtree = make_rtree<RTreeT>(path, fixture);
RTreeT rtree(nodes_path, leaves_path, fixture->coords); LinearSearchNN<TestData> lsnn(fixture.coords, fixture.edges);
LinearSearchNN<TestData> lsnn(fixture->coords, fixture->edges);
simple_verify_rtree(rtree, fixture->coords, fixture->edges); simple_verify_rtree(rtree, fixture.coords, fixture.edges);
sampling_verify_rtree(rtree, lsnn, fixture->coords, 100); sampling_verify_rtree(rtree, lsnn, fixture.coords, 100);
} }
BOOST_FIXTURE_TEST_CASE(construct_tiny, TestRandomGraphFixture_10_30) BOOST_FIXTURE_TEST_CASE(construct_tiny, TestRandomGraphFixture_10_30)
{ {
using TinyTestTree = StaticRTree<TestData, osrm::storage::Ownership::Container, 2, 64>; using TinyTestTree = StaticRTree<TestData, osrm::storage::Ownership::Container, 2, 64>;
construction_test<TinyTestTree>("test_tiny", this); construction_test<TinyTestTree>("test_tiny", *this);
} }
BOOST_FIXTURE_TEST_CASE(construct_half_leaf_test, TestRandomGraphFixture_LeafHalfFull) BOOST_FIXTURE_TEST_CASE(construct_half_leaf_test, TestRandomGraphFixture_LeafHalfFull)
{ {
construction_test("test_1", this); construction_test("test_1", *this);
} }
BOOST_FIXTURE_TEST_CASE(construct_full_leaf_test, TestRandomGraphFixture_LeafFull) BOOST_FIXTURE_TEST_CASE(construct_full_leaf_test, TestRandomGraphFixture_LeafFull)
{ {
construction_test("test_2", this); construction_test("test_2", *this);
} }
BOOST_FIXTURE_TEST_CASE(construct_two_leaves_test, TestRandomGraphFixture_TwoLeaves) BOOST_FIXTURE_TEST_CASE(construct_two_leaves_test, TestRandomGraphFixture_TwoLeaves)
{ {
construction_test("test_3", this); construction_test("test_3", *this);
} }
BOOST_FIXTURE_TEST_CASE(construct_branch_test, TestRandomGraphFixture_Branch) BOOST_FIXTURE_TEST_CASE(construct_branch_test, TestRandomGraphFixture_Branch)
{ {
construction_test("test_4", this); construction_test("test_4", *this);
} }
BOOST_FIXTURE_TEST_CASE(construct_multiple_levels_test, TestRandomGraphFixture_MultipleLevels) BOOST_FIXTURE_TEST_CASE(construct_multiple_levels_test, TestRandomGraphFixture_MultipleLevels)
{ {
construction_test("test_5", this); construction_test("test_5", *this);
} }
// Bug: If you querry a point that lies between two BBs that have a gap, // Bug: If you querry a point that lies between two BBs that have a gap,
@ -321,11 +315,8 @@ BOOST_AUTO_TEST_CASE(regression_test)
}, },
{Edge(0, 1), Edge(2, 3), Edge(4, 5), Edge(6, 7), Edge(8, 9)}); {Edge(0, 1), Edge(2, 3), Edge(4, 5), Edge(6, 7), Edge(8, 9)});
std::string leaves_path; TemporaryFile tmp;
std::string nodes_path; auto rtree = make_rtree<MiniStaticRTree>(tmp.path, fixture);
build_rtree<GraphFixture, MiniStaticRTree>(
"test_regression", &fixture, leaves_path, nodes_path);
MiniStaticRTree rtree(nodes_path, leaves_path, fixture.coords);
LinearSearchNN<TestData> lsnn(fixture.coords, fixture.edges); LinearSearchNN<TestData> lsnn(fixture.coords, fixture.edges);
// query a node just right of the center of the gap // query a node just right of the center of the gap
@ -352,10 +343,8 @@ BOOST_AUTO_TEST_CASE(radius_regression_test)
}, },
{Edge(0, 1), Edge(1, 0)}); {Edge(0, 1), Edge(1, 0)});
std::string leaves_path; TemporaryFile tmp;
std::string nodes_path; auto rtree = make_rtree<MiniStaticRTree>(tmp.path, fixture);
build_rtree<GraphFixture, MiniStaticRTree>("test_angle", &fixture, leaves_path, nodes_path);
MiniStaticRTree rtree(nodes_path, leaves_path, fixture.coords);
TestDataFacade mockfacade; TestDataFacade mockfacade;
engine::GeospatialQuery<MiniStaticRTree, TestDataFacade> query( engine::GeospatialQuery<MiniStaticRTree, TestDataFacade> query(
rtree, fixture.coords, mockfacade); rtree, fixture.coords, mockfacade);
@ -380,10 +369,8 @@ BOOST_AUTO_TEST_CASE(bearing_tests)
}, },
{Edge(0, 1), Edge(1, 0)}); {Edge(0, 1), Edge(1, 0)});
std::string leaves_path; TemporaryFile tmp;
std::string nodes_path; auto rtree = make_rtree<MiniStaticRTree>(tmp.path, fixture);
build_rtree<GraphFixture, MiniStaticRTree>("test_bearing", &fixture, leaves_path, nodes_path);
MiniStaticRTree rtree(nodes_path, leaves_path, fixture.coords);
TestDataFacade mockfacade; TestDataFacade mockfacade;
engine::GeospatialQuery<MiniStaticRTree, TestDataFacade> query( engine::GeospatialQuery<MiniStaticRTree, TestDataFacade> query(
rtree, fixture.coords, mockfacade); rtree, fixture.coords, mockfacade);
@ -459,10 +446,8 @@ BOOST_AUTO_TEST_CASE(bbox_search_tests)
}, },
{Edge(0, 1), Edge(1, 2), Edge(2, 3), Edge(3, 4)}); {Edge(0, 1), Edge(1, 2), Edge(2, 3), Edge(3, 4)});
std::string leaves_path; TemporaryFile tmp;
std::string nodes_path; auto rtree = make_rtree<MiniStaticRTree>(tmp.path, fixture);
build_rtree<GraphFixture, MiniStaticRTree>("test_bbox", &fixture, leaves_path, nodes_path);
MiniStaticRTree rtree(nodes_path, leaves_path, fixture.coords);
TestDataFacade mockfacade; TestDataFacade mockfacade;
engine::GeospatialQuery<MiniStaticRTree, TestDataFacade> query( engine::GeospatialQuery<MiniStaticRTree, TestDataFacade> query(
rtree, fixture.coords, mockfacade); rtree, fixture.coords, mockfacade);