Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Port .ramIndex to tar file and mmap .fileIndex directly

Browse Source
This commit is contained in:
Patrick Niklaus
2018-03-21 17:09:59 +00:00
parent 86ffce3a50
commit 8152dcfb4c
10 changed files with 205 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/benchmarks/CMakeLists.txt
+1 -1
View File
@@ -6,7 +6,7 @@ file(GLOB PackedVectorBenchmarkSources packed_vector.cpp)
add_executable(rtree-bench
EXCLUDE_FROM_ALL
${RTreeBenchmarkSources}
$<TARGET_OBJECTS:UTIL>)
$<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
target_include_directories(rtree-bench
PUBLIC
src/benchmarks/static_rtree.cpp
+8 -5
View File
@@ -1,6 +1,9 @@
#include "util/static_rtree.hpp"
#include "extractor/edge_based_node_segment.hpp"
#include "extractor/query_node.hpp"
#include "extractor/files.hpp"
#include "extractor/packed_osm_ids.hpp"
#include "mocks/mock_datafacade.hpp"
#include "storage/io.hpp"
#include "engine/geospatial_query.hpp"
@@ -32,11 +35,10 @@ using BenchStaticRTree = util::StaticRTree<RTreeLeaf, storage::Ownership::Contai
std::vector<util::Coordinate> loadCoordinates(const boost::filesystem::path &nodes_file)
{
storage::io::FileReader nodes_path_file_reader(nodes_file,
storage::io::FileReader::VerifyFingerprint);
std::vector<util::Coordinate> coords;
storage::serialization::read(nodes_path_file_reader, coords);
extractor::PackedOSMIDs nodes;
extractor::files::readNodes(nodes_file, coords, nodes);
return coords;
}
@@ -99,7 +101,8 @@ int main(int argc, char **argv)
auto coords = osrm::benchmarks::loadCoordinates(nodes_path);
osrm::benchmarks::BenchStaticRTree rtree(ram_path, file_path, coords);
osrm::benchmarks::BenchStaticRTree rtree(file_path, coords);
osrm::extractor::files::readRamIndex(ram_path, rtree);
osrm::benchmarks::benchmark(rtree, 10000);
src/extractor/extractor.cpp
+4 -4
View File
@@ -805,10 +805,10 @@ void Extractor::BuildRTree(std::vector<EdgeBasedNodeSegment> edge_based_node_seg
edge_based_node_segments.resize(new_size);
TIMER_START(construction);
util::StaticRTree<EdgeBasedNodeSegment> rtree(edge_based_node_segments,
config.GetPath(".osrm.ramIndex").string(),
config.GetPath(".osrm.fileIndex").string(),
coordinates);
util::StaticRTree<EdgeBasedNodeSegment> rtree(edge_based_node_segments, coordinates,
config.GetPath(".osrm.fileIndex"));
files::writeRamIndex(config.GetPath(".osrm.ramIndex"), rtree);
TIMER_STOP(construction);
util::Log() << "finished r-tree construction in " << TIMER_SEC(construction) << " seconds";
src/storage/storage.cpp
+19 -24
View File
@@ -244,19 +244,6 @@ void Storage::PopulateLayout(DataLayout &layout)
make_block<char>(absolute_file_index_path.string().length() + 1));
}
// load rsearch tree size
{
io::FileReader tree_node_file(config.GetPath(".osrm.ramIndex"),
io::FileReader::VerifyFingerprint);
const auto tree_size = tree_node_file.ReadElementCount64();
layout.SetBlock(DataLayout::R_SEARCH_TREE, make_block<RTreeNode>(tree_size));
tree_node_file.Skip<RTreeNode>(tree_size);
const auto tree_levels_size = tree_node_file.ReadElementCount64();
layout.SetBlock(DataLayout::R_SEARCH_TREE_LEVELS,
make_block<std::uint64_t>(tree_levels_size));
}
std::unordered_map<std::string, DataLayout::BlockID> name_to_block_id = {
{"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST},
{"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST},
@@ -333,6 +320,8 @@ void Storage::PopulateLayout(DataLayout &layout)
{"/common/turn_data/connectivity_checksum", DataLayout::IGNORE_BLOCK},
{"/common/names/blocks", DataLayout::NAME_BLOCKS},
{"/common/names/values", DataLayout::NAME_VALUES},
{"/common/rtree/search_tree", DataLayout::R_SEARCH_TREE},
{"/common/rtree/search_tree_level_starts", DataLayout::R_SEARCH_TREE_LEVEL_STARTS},
};
std::vector<NamedBlock> blocks;
@@ -357,6 +346,7 @@ void Storage::PopulateLayout(DataLayout &layout)
{REQUIRED, config.GetPath(".osrm.turn_duration_penalties")},
{REQUIRED, config.GetPath(".osrm.edges")},
{REQUIRED, config.GetPath(".osrm.names")},
{REQUIRED, config.GetPath(".osrm.ramIndex")},
};
for (const auto &file : tar_files)
@@ -636,22 +626,27 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// store search tree portion of rtree
{
io::FileReader tree_node_file(config.GetPath(".osrm.ramIndex"),
io::FileReader::VerifyFingerprint);
// perform this read so that we're at the right stream position for the next
// read.
tree_node_file.Skip<std::uint64_t>(1);
const auto rtree_ptr =
layout.GetBlockPtr<RTreeNode, true>(memory_ptr, DataLayout::R_SEARCH_TREE);
util::vector_view<RTreeNode> search_tree(
rtree_ptr, layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE));
tree_node_file.ReadInto(rtree_ptr, layout.GetBlockEntries(DataLayout::R_SEARCH_TREE));
const auto rtree_levelstarts_ptr = layout.GetBlockPtr<std::uint64_t, true>(
memory_ptr, DataLayout::R_SEARCH_TREE_LEVEL_STARTS);
util::vector_view<std::uint64_t> rtree_level_starts(
rtree_levelstarts_ptr,
layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS));
tree_node_file.Skip<std::uint64_t>(1);
const auto rtree_levelsizes_ptr =
layout.GetBlockPtr<std::uint64_t, true>(memory_ptr, DataLayout::R_SEARCH_TREE_LEVELS);
// we need this purely for the interface
util::vector_view<util::Coordinate> empty_coords;
tree_node_file.ReadInto(rtree_levelsizes_ptr,
layout.GetBlockEntries(DataLayout::R_SEARCH_TREE_LEVELS));
util::StaticRTree<RTreeLeaf, storage::Ownership::View> rtree{
std::move(search_tree),
std::move(rtree_level_starts),
config.GetPath(".osrm.fileIndex"),
empty_coords};
extractor::files::readRamIndex(config.GetPath(".osrm.ramIndex"), rtree);
}
// load profile properties