Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Integrate CellStorage into datafacade

Browse Source
This commit is contained in:
Patrick Niklaus
2017-03-04 18:36:29 +00:00
committed by Patrick Niklaus
parent ef71cc0d12
commit 694bf9d8b1
9 changed files with 157 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/engine/datafacade/algorithm_datafacade.hpp
+1 -1
View File
@@ -66,7 +66,7 @@ template <> class AlgorithmDataFacade<algorithm::MLD>
public:
virtual const util::MultiLevelPartitionView &GetMultiLevelPartition() const = 0;
virtual const util::CellStorage<true> &GetCellStorage() const = 0;
virtual const util::CellStorageView &GetCellStorage() const = 0;
};
}
}
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+46 -9
View File
@@ -1089,7 +1089,7 @@ class ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>
{
// MLD data
util::MultiLevelPartitionView mld_partition;
util::CellStorage<true> mld_cell_storage;
util::CellStorageView mld_cell_storage;
void InitializeInternalPointers(storage::DataLayout &data_layout, char *memory_block)
{
@@ -1123,14 +1123,51 @@ class ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>
mld_partition = util::MultiLevelPartitionView{level_data, partition, cell_to_children};
}
if (data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_STORAGE) > 0)
if (data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_WEIGHTS) > 0)
{
auto mld_cell_storage_ptr =
data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::MLD_CELL_STORAGE);
mld_cell_storage.InitializePointers(
mld_cell_storage_ptr,
mld_cell_storage_ptr +
data_layout.num_entries[storage::DataLayout::MLD_CELL_STORAGE]);
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY) >
0);
BOOST_ASSERT(
data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY) > 0);
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0);
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0);
auto mld_cell_weights_ptr = data_layout.GetBlockPtr<EdgeWeight>(
memory_block, storage::DataLayout::MLD_CELL_WEIGHTS);
auto mld_source_boundary_ptr = data_layout.GetBlockPtr<NodeID>(
memory_block, storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
auto mld_destination_boundary_ptr = data_layout.GetBlockPtr<NodeID>(
memory_block, storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY);
auto mld_cells_ptr = data_layout.GetBlockPtr<util::CellStorageView::CellData>(
memory_block, storage::DataLayout::MLD_CELLS);
auto mld_cell_level_offsets_ptr = data_layout.GetBlockPtr<std::uint64_t>(
memory_block, storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
auto weight_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_WEIGHTS);
auto source_boundary_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
auto destination_boundary_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY);
auto cells_entries_counts = data_layout.GetBlockEntries(storage::DataLayout::MLD_CELLS);
auto cell_level_offsets_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
util::ShM<EdgeWeight, true>::vector weights(mld_cell_weights_ptr, weight_entries_count);
util::ShM<NodeID, true>::vector source_boundary(mld_source_boundary_ptr,
source_boundary_entries_count);
util::ShM<NodeID, true>::vector destination_boundary(
mld_destination_boundary_ptr, destination_boundary_entries_count);
util::ShM<util::CellStorageView::CellData, true>::vector cells(mld_cells_ptr,
cells_entries_counts);
util::ShM<std::uint64_t, true>::vector level_offsets(mld_cell_level_offsets_ptr,
cell_level_offsets_entries_count);
mld_cell_storage = util::CellStorageView{std::move(weights),
std::move(source_boundary),
std::move(destination_boundary),
std::move(cells),
std::move(level_offsets)};
}
}
@@ -1147,7 +1184,7 @@ class ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>
const util::MultiLevelPartitionView &GetMultiLevelPartition() const { return mld_partition; }
const util::CellStorage<true> &GetCellStorage() const { return mld_cell_storage; }
const util::CellStorageView &GetCellStorage() const { return mld_cell_storage; }
};
template <>
include/partition/io.hpp
+15
View File
@@ -3,6 +3,7 @@
#include "storage/io.hpp"
#include "util/multi_level_partition.hpp"
#include "util/cell_storage.hpp"
namespace osrm
{
@@ -21,6 +22,20 @@ inline void write(const boost::filesystem::path &path, const util::MultiLevelPar
writer.SerializeVector(mlp.partition);
writer.SerializeVector(mlp.cell_to_children);
}
template <>
inline void write(const boost::filesystem::path &path, const util::CellStorage &storage)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(storage.weights);
writer.SerializeVector(storage.source_boundary);
writer.SerializeVector(storage.destination_boundary);
writer.SerializeVector(storage.cells);
writer.SerializeVector(storage.level_to_cell_offset);
}
}
}
}
include/storage/shared_datatype.hpp
+10 -2
View File
@@ -60,7 +60,11 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
"MLD_LEVEL_DATA",
"MLD_PARTITION",
"MLD_CELL_TO_CHILDREN",
"MLD_CELL_STORAGE"};
"MLD_CELL_WEIGHTS",
"MLD_CELL_SOURCE_BOUNDARY",
"MLD_CELL_DESTINATION_BOUNDARY",
"MLD_CELLS",
"MLD_CELL_LEVEL_OFFSETS"};
struct DataLayout
{
@@ -108,7 +112,11 @@ struct DataLayout
MLD_LEVEL_DATA,
MLD_PARTITION,
MLD_CELL_TO_CHILDREN,
MLD_CELL_STORAGE,
MLD_CELL_WEIGHTS,
MLD_CELL_SOURCE_BOUNDARY,
MLD_CELL_DESTINATION_BOUNDARY,
MLD_CELLS,
MLD_CELL_LEVEL_OFFSETS,
NUM_BLOCKS
};
include/util/cell_storage.hpp
+37 -104
View File
@@ -22,8 +22,24 @@ namespace osrm
{
namespace util
{
namespace detail {
template <bool UseShareMemory> class CellStorageImpl;
}
using CellStorage = detail::CellStorageImpl<false>;
using CellStorageView = detail::CellStorageImpl<true>;
}
namespace partition {
namespace io {
template <bool UseShareMemory>
inline void write(const boost::filesystem::path &path, const util::detail::CellStorageImpl<UseShareMemory> &storage);
}
}
template <bool UseShareMemory> class CellStorage
namespace util
{
namespace detail
{
template <bool UseShareMemory> class CellStorageImpl
{
public:
using WeightOffset = std::uint32_t;
@@ -35,7 +51,6 @@ template <bool UseShareMemory> class CellStorage
static constexpr auto INVALID_WEIGHT_OFFSET = std::numeric_limits<WeightOffset>::max();
static constexpr auto INVALID_BOUNDARY_OFFSET = std::numeric_limits<BoundaryOffset>::max();
private:
struct CellData
{
WeightOffset weight_offset = INVALID_WEIGHT_OFFSET;
@@ -45,6 +60,9 @@ template <bool UseShareMemory> class CellStorage
BoundarySize num_destination_nodes = 0;
};
private:
template <typename T> using Vector = typename util::ShM<T, UseShareMemory>::vector;
// Implementation of the cell view. We need a template parameter here
// because we need to derive a read-only and read-write view from this.
template <typename WeightValueT> class CellImpl
@@ -165,10 +183,10 @@ template <bool UseShareMemory> class CellStorage
using Cell = CellImpl<EdgeWeight>;
using ConstCell = CellImpl<const EdgeWeight>;
CellStorage() {}
CellStorageImpl() {}
template <typename GraphT>
CellStorage(const MultiLevelPartition &partition, const GraphT &base_graph)
template <typename GraphT, typename = std::enable_if<!UseShareMemory>>
CellStorageImpl(const MultiLevelPartition &partition, const GraphT &base_graph)
{
// pre-allocate storge for CellData so we can have random access to it by cell id
unsigned number_of_cells = 0;
@@ -281,19 +299,18 @@ template <bool UseShareMemory> class CellStorage
weights.resize(weight_offset + 1, INVALID_EDGE_WEIGHT);
}
CellStorage(std::vector<EdgeWeight> weights_,
std::vector<NodeID> source_boundary_,
std::vector<NodeID> destination_boundary_,
std::vector<CellData> cells_,
std::vector<std::size_t> level_to_cell_offset_)
template <typename = std::enable_if<UseShareMemory>>
CellStorageImpl(Vector<EdgeWeight> weights_,
Vector<NodeID> source_boundary_,
Vector<NodeID> destination_boundary_,
Vector<CellData> cells_,
Vector<std::uint64_t> level_to_cell_offset_)
: weights(std::move(weights_)), source_boundary(std::move(source_boundary_)),
destination_boundary(std::move(destination_boundary_)), cells(std::move(cells_)),
level_to_cell_offset(std::move(level_to_cell_offset_))
{
}
CellStorage(const boost::filesystem::path &path) { Read(path); }
ConstCell GetCell(LevelID level, CellID id) const
{
const auto level_index = LevelIDToIndex(level);
@@ -305,6 +322,7 @@ template <bool UseShareMemory> class CellStorage
cells[cell_index], weights.data(), source_boundary.data(), destination_boundary.data()};
}
template <typename = std::enable_if<!UseShareMemory>>
Cell GetCell(LevelID level, CellID id)
{
const auto level_index = LevelIDToIndex(level);
@@ -316,102 +334,17 @@ template <bool UseShareMemory> class CellStorage
cells[cell_index], weights.data(), source_boundary.data(), destination_boundary.data()};
}
std::size_t GetRequiredMemorySize(const boost::filesystem::path &path) const
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
std::size_t memory_size = 0;
memory_size += reader.GetVectorMemorySize<decltype(weights[0])>();
memory_size += reader.GetVectorMemorySize<decltype(source_boundary[0])>();
memory_size += reader.GetVectorMemorySize<decltype(destination_boundary[0])>();
memory_size += reader.GetVectorMemorySize<decltype(cells[0])>();
memory_size += reader.GetVectorMemorySize<decltype(level_to_cell_offset[0])>();
return memory_size;
}
template <bool Q = UseShareMemory>
typename std::enable_if<Q>::type
Read(const boost::filesystem::path &path, void *begin, const void *end) const
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
begin = reader.DeserializeVector<typename decltype(weights)::value_type>(begin, end);
begin =
reader.DeserializeVector<typename decltype(source_boundary)::value_type>(begin, end);
begin = reader.DeserializeVector<typename decltype(destination_boundary)::value_type>(begin,
end);
begin = reader.DeserializeVector<typename decltype(cells)::value_type>(begin, end);
begin = reader.DeserializeVector<typename decltype(level_to_cell_offset)::value_type>(begin,
end);
}
template <bool Q = UseShareMemory>
typename std::enable_if<Q>::type InitializePointers(char *begin, const char *end)
{
auto weights_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(weights_size);
weights.reset(reinterpret_cast<EdgeWeight *>(begin), weights_size);
begin += sizeof(decltype(weights[0])) * weights_size;
auto source_boundary_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(source_boundary_size);
source_boundary.reset(reinterpret_cast<NodeID *>(begin), source_boundary_size);
begin += sizeof(decltype(source_boundary[0])) * source_boundary_size;
auto destination_boundary_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(destination_boundary_size);
destination_boundary.reset(reinterpret_cast<NodeID *>(begin), destination_boundary_size);
begin += sizeof(decltype(destination_boundary[0])) * destination_boundary_size;
auto cells_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(cells_size);
cells.reset(reinterpret_cast<CellData *>(begin), cells_size);
begin += sizeof(decltype(cells[0])) * cells_size;
auto level_to_cell_offset_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(level_to_cell_offset_size);
level_to_cell_offset.reset(reinterpret_cast<std::size_t *>(begin),
level_to_cell_offset_size);
begin += sizeof(decltype(level_to_cell_offset[0])) * level_to_cell_offset_size;
BOOST_ASSERT(begin <= end);
}
template <bool Q = UseShareMemory>
typename std::enable_if<!Q>::type Read(const boost::filesystem::path &path)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(weights);
reader.DeserializeVector(source_boundary);
reader.DeserializeVector(destination_boundary);
reader.DeserializeVector(cells);
reader.DeserializeVector(level_to_cell_offset);
}
void Write(const boost::filesystem::path &path) const
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(weights);
writer.SerializeVector(source_boundary);
writer.SerializeVector(destination_boundary);
writer.SerializeVector(cells);
writer.SerializeVector(level_to_cell_offset);
}
friend void partition::io::write<UseShareMemory>(const boost::filesystem::path &path, const util::detail::CellStorageImpl<UseShareMemory> &storage);
private:
typename util::ShM<EdgeWeight, UseShareMemory>::vector weights;
typename util::ShM<NodeID, UseShareMemory>::vector source_boundary;
typename util::ShM<NodeID, UseShareMemory>::vector destination_boundary;
typename util::ShM<CellData, UseShareMemory>::vector cells;
typename util::ShM<std::size_t, UseShareMemory>::vector level_to_cell_offset;
Vector<EdgeWeight> weights;
Vector<NodeID> source_boundary;
Vector<NodeID> destination_boundary;
Vector<CellData> cells;
Vector<std::uint64_t> level_to_cell_offset;
};
}
}
}
#endif