Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Initial interation on writing out MLD partition/cell data

Browse Source
This commit is contained in:
Michael Krasnyk
2017-02-23 16:39:29 +01:00
committed by Patrick Niklaus
parent 757e7ca936
commit ff0a98196f
15 changed files with 415 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/util/cell_storage.hpp
+102 -6
View File
@@ -3,9 +3,13 @@
#include "util/assert.hpp"
#include "util/for_each_range.hpp"
#include "util/log.hpp"
#include "util/multi_level_partition.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/typedefs.hpp"
#include "storage/io.hpp"
#include <boost/range/iterator_range.hpp>
#include <tbb/parallel_sort.h>
@@ -19,7 +23,7 @@ namespace osrm
namespace util
{
class CellStorage
template <bool UseShareMemory> class CellStorage
{
public:
using WeightOffset = std::uint32_t;
@@ -161,6 +165,8 @@ class CellStorage
using Cell = CellImpl<EdgeWeight>;
using ConstCell = CellImpl<const EdgeWeight>;
CellStorage() {}
template <typename GraphT>
CellStorage(const MultiLevelPartition &partition, const GraphT &base_graph)
{
@@ -286,6 +292,8 @@ class CellStorage
{
}
CellStorage(const boost::filesystem::path &path) { Read(path); }
ConstCell GetCell(LevelID level, CellID id) const
{
const auto level_index = LevelIDToIndex(level);
@@ -308,12 +316,100 @@ class CellStorage
cells[cell_index], weights.data(), source_boundary.data(), destination_boundary.data()};
}
std::size_t GetRequiredMemorySize(const boost::filesystem::path &path) const
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
std::size_t memory_size = 0;
memory_size += reader.GetVectorMemorySize<decltype(weights[0])>();
memory_size += reader.GetVectorMemorySize<decltype(source_boundary[0])>();
memory_size += reader.GetVectorMemorySize<decltype(destination_boundary[0])>();
memory_size += reader.GetVectorMemorySize<decltype(cells[0])>();
memory_size += reader.GetVectorMemorySize<decltype(level_to_cell_offset[0])>();
return memory_size;
}
template <bool Q = UseShareMemory>
typename std::enable_if<Q>::type
Read(const boost::filesystem::path &path, void *begin, const void *end) const
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
begin = reader.DeserializeVector<typename decltype(weights)::value_type>(begin, end);
begin =
reader.DeserializeVector<typename decltype(source_boundary)::value_type>(begin, end);
begin = reader.DeserializeVector<typename decltype(destination_boundary)::value_type>(begin,
end);
begin = reader.DeserializeVector<typename decltype(cells)::value_type>(begin, end);
begin = reader.DeserializeVector<typename decltype(level_to_cell_offset)::value_type>(begin,
end);
}
template <bool Q = UseShareMemory>
typename std::enable_if<Q>::type InitializePointers(char *begin, const char *end)
{
auto weights_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(weights_size);
weights.reset(reinterpret_cast<EdgeWeight *>(begin), weights_size);
begin += sizeof(decltype(weights[0])) * weights_size;
auto source_boundary_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(source_boundary_size);
source_boundary.reset(reinterpret_cast<NodeID *>(begin), source_boundary_size);
begin += sizeof(decltype(source_boundary[0])) * source_boundary_size;
auto destination_boundary_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(destination_boundary_size);
destination_boundary.reset(reinterpret_cast<NodeID *>(begin), destination_boundary_size);
begin += sizeof(decltype(destination_boundary[0])) * destination_boundary_size;
auto cells_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(cells_size);
cells.reset(reinterpret_cast<CellData *>(begin), cells_size);
begin += sizeof(decltype(cells[0])) * cells_size;
auto level_to_cell_offset_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(level_to_cell_offset_size);
level_to_cell_offset.reset(reinterpret_cast<std::size_t *>(begin),
level_to_cell_offset_size);
begin += sizeof(decltype(level_to_cell_offset[0])) * level_to_cell_offset_size;
BOOST_ASSERT(begin <= end);
}
template <bool Q = UseShareMemory>
typename std::enable_if<!Q>::type Read(const boost::filesystem::path &path)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(weights);
reader.DeserializeVector(source_boundary);
reader.DeserializeVector(destination_boundary);
reader.DeserializeVector(cells);
reader.DeserializeVector(level_to_cell_offset);
}
void Write(const boost::filesystem::path &path) const
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(weights);
writer.SerializeVector(source_boundary);
writer.SerializeVector(destination_boundary);
writer.SerializeVector(cells);
writer.SerializeVector(level_to_cell_offset);
}
private:
std::vector<EdgeWeight> weights;
std::vector<NodeID> source_boundary;
std::vector<NodeID> destination_boundary;
std::vector<CellData> cells;
std::vector<std::size_t> level_to_cell_offset;
typename util::ShM<EdgeWeight, UseShareMemory>::vector weights;
typename util::ShM<NodeID, UseShareMemory>::vector source_boundary;
typename util::ShM<NodeID, UseShareMemory>::vector destination_boundary;
typename util::ShM<CellData, UseShareMemory>::vector cells;
typename util::ShM<std::size_t, UseShareMemory>::vector level_to_cell_offset;
};
}
}
include/util/multi_level_partition.hpp
+95 -5
View File
@@ -3,8 +3,11 @@
#include "util/exception.hpp"
#include "util/for_each_pair.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/typedefs.hpp"
#include "storage/io.hpp"
#include <algorithm>
#include <array>
#include <climits>
@@ -73,12 +76,14 @@ class MultiLevelPartition
virtual std::uint32_t GetNumberOfCells(LevelID level) const = 0;
};
class PackedMultiLevelPartition final : public MultiLevelPartition
template <bool UseShareMemory> class PackedMultiLevelPartition final : public MultiLevelPartition
{
using PartitionID = std::uint64_t;
static const constexpr std::uint8_t NUM_PARTITION_BITS = sizeof(PartitionID) * CHAR_BIT;
public:
PackedMultiLevelPartition() {}
// cell_sizes is index by level (starting at 0, the base graph).
// However level 0 always needs to have cell size 1, since it is the
// basegraph.
@@ -90,6 +95,8 @@ class PackedMultiLevelPartition final : public MultiLevelPartition
initializePartitionIDs(partitions);
}
PackedMultiLevelPartition(const boost::filesystem::path &file_name) { Read(file_name); }
// returns the index of the cell the vertex is contained at level l
CellID GetCell(LevelID l, NodeID node) const final override
{
@@ -139,6 +146,89 @@ class PackedMultiLevelPartition final : public MultiLevelPartition
return cell_to_children[offset + cell + 1];
}
std::size_t GetRequiredMemorySize(const boost::filesystem::path &path) const
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
std::size_t memory_size = 0;
memory_size += reader.GetVectorMemorySize<decltype(level_offsets[0])>();
memory_size += reader.GetVectorMemorySize<decltype(partition[0])>();
memory_size += reader.GetVectorMemorySize<decltype(level_to_children_offset[0])>();
memory_size += reader.GetVectorMemorySize<decltype(cell_to_children[0])>();
return memory_size;
}
template <bool Q = UseShareMemory>
typename std::enable_if<Q>::type
Read(const boost::filesystem::path &path, void *begin, const void *end) const
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
begin = reader.DeserializeVector<typename decltype(level_offsets)::value_type>(begin, end);
begin = reader.DeserializeVector<typename decltype(partition)::value_type>(begin, end);
begin = reader.DeserializeVector<typename decltype(level_to_children_offset)::value_type>(
begin, end);
begin =
reader.DeserializeVector<typename decltype(cell_to_children)::value_type>(begin, end);
}
template <bool Q = UseShareMemory>
typename std::enable_if<Q>::type InitializePointers(char *begin, const char *end)
{
auto level_offsets_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(level_offsets_size);
level_offsets.reset(begin, level_offsets_size);
begin += sizeof(decltype(level_offsets[0])) * level_offsets_size;
auto partition_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(partition_size);
partition.reset(begin, partition_size);
begin += sizeof(decltype(partition[0])) * partition_size;
auto level_to_children_offset_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(level_to_children_offset_size);
level_to_children_offset.reset(begin, level_to_children_offset_size);
begin += sizeof(decltype(level_to_children_offset[0])) * level_to_children_offset_size;
auto cell_to_children_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(cell_to_children_size);
cell_to_children.reset(begin, cell_to_children_size);
begin += sizeof(decltype(cell_to_children[0])) * level_to_children_offset_size;
BOOST_ASSERT(begin <= end);
level_masks = makeLevelMasks();
bit_to_level = makeBitToLevel();
}
template <bool Q = UseShareMemory>
typename std::enable_if<!Q>::type Read(const boost::filesystem::path &path)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(level_offsets);
reader.DeserializeVector(partition);
reader.DeserializeVector(level_to_children_offset);
reader.DeserializeVector(cell_to_children);
level_masks = makeLevelMasks();
bit_to_level = makeBitToLevel();
}
void Write(const boost::filesystem::path &path) const
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(level_offsets);
writer.SerializeVector(partition);
writer.SerializeVector(level_to_children_offset);
writer.SerializeVector(cell_to_children);
}
private:
inline std::size_t LevelIDToIndex(LevelID l) const { return l - 1; }
@@ -306,11 +396,11 @@ class PackedMultiLevelPartition final : public MultiLevelPartition
}
}
std::vector<PartitionID> partition;
std::vector<std::uint8_t> level_offsets;
typename util::ShM<std::uint8_t, UseShareMemory>::vector level_offsets;
typename util::ShM<PartitionID, UseShareMemory>::vector partition;
typename util::ShM<std::uint32_t, UseShareMemory>::vector level_to_children_offset;
typename util::ShM<CellID, UseShareMemory>::vector cell_to_children;
std::vector<PartitionID> level_masks;
std::vector<std::uint32_t> level_to_children_offset;
std::vector<CellID> cell_to_children;
std::array<LevelID, NUM_PARTITION_BITS> bit_to_level;
};
}
include/util/shared_memory_vector_wrapper.hpp
+7
View File
@@ -58,6 +58,7 @@ template <typename DataT> class SharedMemoryWrapper
std::size_t m_size;
public:
using value_type = DataT;
using iterator = ShMemIterator<DataT>;
using reverse_iterator = boost::reverse_iterator<iterator>;
@@ -71,6 +72,12 @@ template <typename DataT> class SharedMemoryWrapper
m_size = size;
}
void reset(void *ptr, std::size_t size)
{
m_ptr = reinterpret_cast<DataT *>(ptr);
m_size = size;
}
DataT &at(const std::size_t index) { return m_ptr[index]; }
const DataT &at(const std::size_t index) const { return m_ptr[index]; }