Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Integrate CellStorage into datafacade

Browse Source
This commit is contained in:
Patrick Niklaus 2017-03-04 18:36:29 +00:00 committed by Patrick Niklaus
parent ef71cc0d12
commit 694bf9d8b1
9 changed files with 157 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/engine/datafacade/algorithm_datafacade.hpp
View File

@ -66,7 +66,7 @@ template <> class AlgorithmDataFacade<algorithm::MLD>
public:
virtual const util::MultiLevelPartitionView &GetMultiLevelPartition() const = 0;
virtual const util::CellStorage<true> &GetCellStorage() const = 0;
virtual const util::CellStorageView &GetCellStorage() const = 0;
};
}
}

55
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View File

@ -1089,7 +1089,7 @@ class ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>
{
// MLD data
util::MultiLevelPartitionView mld_partition;
util::CellStorage<true> mld_cell_storage;
util::CellStorageView mld_cell_storage;
void InitializeInternalPointers(storage::DataLayout &data_layout, char *memory_block)
{
@ -1123,14 +1123,51 @@ class ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>
mld_partition = util::MultiLevelPartitionView{level_data, partition, cell_to_children};
}
if (data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_STORAGE) > 0)
if (data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_WEIGHTS) > 0)
{
auto mld_cell_storage_ptr =
data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::MLD_CELL_STORAGE);
mld_cell_storage.InitializePointers(
mld_cell_storage_ptr,
mld_cell_storage_ptr +
data_layout.num_entries[storage::DataLayout::MLD_CELL_STORAGE]);
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY) >
0);
BOOST_ASSERT(
data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY) > 0);
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0);
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0);
auto mld_cell_weights_ptr = data_layout.GetBlockPtr<EdgeWeight>(
memory_block, storage::DataLayout::MLD_CELL_WEIGHTS);
auto mld_source_boundary_ptr = data_layout.GetBlockPtr<NodeID>(
memory_block, storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
auto mld_destination_boundary_ptr = data_layout.GetBlockPtr<NodeID>(
memory_block, storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY);
auto mld_cells_ptr = data_layout.GetBlockPtr<util::CellStorageView::CellData>(
memory_block, storage::DataLayout::MLD_CELLS);
auto mld_cell_level_offsets_ptr = data_layout.GetBlockPtr<std::uint64_t>(
memory_block, storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
auto weight_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_WEIGHTS);
auto source_boundary_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
auto destination_boundary_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY);
auto cells_entries_counts = data_layout.GetBlockEntries(storage::DataLayout::MLD_CELLS);
auto cell_level_offsets_entries_count =
data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
util::ShM<EdgeWeight, true>::vector weights(mld_cell_weights_ptr, weight_entries_count);
util::ShM<NodeID, true>::vector source_boundary(mld_source_boundary_ptr,
source_boundary_entries_count);
util::ShM<NodeID, true>::vector destination_boundary(
mld_destination_boundary_ptr, destination_boundary_entries_count);
util::ShM<util::CellStorageView::CellData, true>::vector cells(mld_cells_ptr,
cells_entries_counts);
util::ShM<std::uint64_t, true>::vector level_offsets(mld_cell_level_offsets_ptr,
cell_level_offsets_entries_count);
mld_cell_storage = util::CellStorageView{std::move(weights),
std::move(source_boundary),
std::move(destination_boundary),
std::move(cells),
std::move(level_offsets)};
}
}
@ -1147,7 +1184,7 @@ class ContiguousInternalMemoryAlgorithmDataFacade<algorithm::MLD>
const util::MultiLevelPartitionView &GetMultiLevelPartition() const { return mld_partition; }
const util::CellStorage<true> &GetCellStorage() const { return mld_cell_storage; }
const util::CellStorageView &GetCellStorage() const { return mld_cell_storage; }
};
template <>

15
include/partition/io.hpp
View File

@ -3,6 +3,7 @@
#include "storage/io.hpp"
#include "util/multi_level_partition.hpp"
#include "util/cell_storage.hpp"
namespace osrm
{
@ -21,6 +22,20 @@ inline void write(const boost::filesystem::path &path, const util::MultiLevelPar
writer.SerializeVector(mlp.partition);
writer.SerializeVector(mlp.cell_to_children);
}
template <>
inline void write(const boost::filesystem::path &path, const util::CellStorage &storage)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(storage.weights);
writer.SerializeVector(storage.source_boundary);
writer.SerializeVector(storage.destination_boundary);
writer.SerializeVector(storage.cells);
writer.SerializeVector(storage.level_to_cell_offset);
}
}
}
}

12
include/storage/shared_datatype.hpp
View File

@ -60,7 +60,11 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
"MLD_LEVEL_DATA",
"MLD_PARTITION",
"MLD_CELL_TO_CHILDREN",
"MLD_CELL_STORAGE"};
"MLD_CELL_WEIGHTS",
"MLD_CELL_SOURCE_BOUNDARY",
"MLD_CELL_DESTINATION_BOUNDARY",
"MLD_CELLS",
"MLD_CELL_LEVEL_OFFSETS"};
struct DataLayout
{
@ -108,7 +112,11 @@ struct DataLayout
MLD_LEVEL_DATA,
MLD_PARTITION,
MLD_CELL_TO_CHILDREN,
MLD_CELL_STORAGE,
MLD_CELL_WEIGHTS,
MLD_CELL_SOURCE_BOUNDARY,
MLD_CELL_DESTINATION_BOUNDARY,
MLD_CELLS,
MLD_CELL_LEVEL_OFFSETS,
NUM_BLOCKS
};

141
include/util/cell_storage.hpp
View File

@ -22,8 +22,24 @@ namespace osrm
{
namespace util
{
namespace detail {
template <bool UseShareMemory> class CellStorageImpl;
}
using CellStorage = detail::CellStorageImpl<false>;
using CellStorageView = detail::CellStorageImpl<true>;
}
namespace partition {
namespace io {
template <bool UseShareMemory>
inline void write(const boost::filesystem::path &path, const util::detail::CellStorageImpl<UseShareMemory> &storage);
}
}
template <bool UseShareMemory> class CellStorage
namespace util
{
namespace detail
{
template <bool UseShareMemory> class CellStorageImpl
{
public:
using WeightOffset = std::uint32_t;
@ -35,7 +51,6 @@ template <bool UseShareMemory> class CellStorage
static constexpr auto INVALID_WEIGHT_OFFSET = std::numeric_limits<WeightOffset>::max();
static constexpr auto INVALID_BOUNDARY_OFFSET = std::numeric_limits<BoundaryOffset>::max();
private:
struct CellData
{
WeightOffset weight_offset = INVALID_WEIGHT_OFFSET;
@ -45,6 +60,9 @@ template <bool UseShareMemory> class CellStorage
BoundarySize num_destination_nodes = 0;
};
private:
template <typename T> using Vector = typename util::ShM<T, UseShareMemory>::vector;
// Implementation of the cell view. We need a template parameter here
// because we need to derive a read-only and read-write view from this.
template <typename WeightValueT> class CellImpl
@ -165,10 +183,10 @@ template <bool UseShareMemory> class CellStorage
using Cell = CellImpl<EdgeWeight>;
using ConstCell = CellImpl<const EdgeWeight>;
CellStorage() {}
CellStorageImpl() {}
template <typename GraphT>
CellStorage(const MultiLevelPartition &partition, const GraphT &base_graph)
template <typename GraphT, typename = std::enable_if<!UseShareMemory>>
CellStorageImpl(const MultiLevelPartition &partition, const GraphT &base_graph)
{
// pre-allocate storge for CellData so we can have random access to it by cell id
unsigned number_of_cells = 0;
@ -281,19 +299,18 @@ template <bool UseShareMemory> class CellStorage
weights.resize(weight_offset + 1, INVALID_EDGE_WEIGHT);
}
CellStorage(std::vector<EdgeWeight> weights_,
std::vector<NodeID> source_boundary_,
std::vector<NodeID> destination_boundary_,
std::vector<CellData> cells_,
std::vector<std::size_t> level_to_cell_offset_)
template <typename = std::enable_if<UseShareMemory>>
CellStorageImpl(Vector<EdgeWeight> weights_,
Vector<NodeID> source_boundary_,
Vector<NodeID> destination_boundary_,
Vector<CellData> cells_,
Vector<std::uint64_t> level_to_cell_offset_)
: weights(std::move(weights_)), source_boundary(std::move(source_boundary_)),
destination_boundary(std::move(destination_boundary_)), cells(std::move(cells_)),
level_to_cell_offset(std::move(level_to_cell_offset_))
{
}
CellStorage(const boost::filesystem::path &path) { Read(path); }
ConstCell GetCell(LevelID level, CellID id) const
{
const auto level_index = LevelIDToIndex(level);
@ -305,6 +322,7 @@ template <bool UseShareMemory> class CellStorage
cells[cell_index], weights.data(), source_boundary.data(), destination_boundary.data()};
}
template <typename = std::enable_if<!UseShareMemory>>
Cell GetCell(LevelID level, CellID id)
{
const auto level_index = LevelIDToIndex(level);
@ -316,102 +334,17 @@ template <bool UseShareMemory> class CellStorage
cells[cell_index], weights.data(), source_boundary.data(), destination_boundary.data()};
}
std::size_t GetRequiredMemorySize(const boost::filesystem::path &path) const
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
std::size_t memory_size = 0;
memory_size += reader.GetVectorMemorySize<decltype(weights[0])>();
memory_size += reader.GetVectorMemorySize<decltype(source_boundary[0])>();
memory_size += reader.GetVectorMemorySize<decltype(destination_boundary[0])>();
memory_size += reader.GetVectorMemorySize<decltype(cells[0])>();
memory_size += reader.GetVectorMemorySize<decltype(level_to_cell_offset[0])>();
return memory_size;
}
template <bool Q = UseShareMemory>
typename std::enable_if<Q>::type
Read(const boost::filesystem::path &path, void *begin, const void *end) const
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
begin = reader.DeserializeVector<typename decltype(weights)::value_type>(begin, end);
begin =
reader.DeserializeVector<typename decltype(source_boundary)::value_type>(begin, end);
begin = reader.DeserializeVector<typename decltype(destination_boundary)::value_type>(begin,
end);
begin = reader.DeserializeVector<typename decltype(cells)::value_type>(begin, end);
begin = reader.DeserializeVector<typename decltype(level_to_cell_offset)::value_type>(begin,
end);
}
template <bool Q = UseShareMemory>
typename std::enable_if<Q>::type InitializePointers(char *begin, const char *end)
{
auto weights_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(weights_size);
weights.reset(reinterpret_cast<EdgeWeight *>(begin), weights_size);
begin += sizeof(decltype(weights[0])) * weights_size;
auto source_boundary_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(source_boundary_size);
source_boundary.reset(reinterpret_cast<NodeID *>(begin), source_boundary_size);
begin += sizeof(decltype(source_boundary[0])) * source_boundary_size;
auto destination_boundary_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(destination_boundary_size);
destination_boundary.reset(reinterpret_cast<NodeID *>(begin), destination_boundary_size);
begin += sizeof(decltype(destination_boundary[0])) * destination_boundary_size;
auto cells_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(cells_size);
cells.reset(reinterpret_cast<CellData *>(begin), cells_size);
begin += sizeof(decltype(cells[0])) * cells_size;
auto level_to_cell_offset_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(level_to_cell_offset_size);
level_to_cell_offset.reset(reinterpret_cast<std::size_t *>(begin),
level_to_cell_offset_size);
begin += sizeof(decltype(level_to_cell_offset[0])) * level_to_cell_offset_size;
BOOST_ASSERT(begin <= end);
}
template <bool Q = UseShareMemory>
typename std::enable_if<!Q>::type Read(const boost::filesystem::path &path)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(weights);
reader.DeserializeVector(source_boundary);
reader.DeserializeVector(destination_boundary);
reader.DeserializeVector(cells);
reader.DeserializeVector(level_to_cell_offset);
}
void Write(const boost::filesystem::path &path) const
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(weights);
writer.SerializeVector(source_boundary);
writer.SerializeVector(destination_boundary);
writer.SerializeVector(cells);
writer.SerializeVector(level_to_cell_offset);
}
friend void partition::io::write<UseShareMemory>(const boost::filesystem::path &path, const util::detail::CellStorageImpl<UseShareMemory> &storage);
private:
typename util::ShM<EdgeWeight, UseShareMemory>::vector weights;
typename util::ShM<NodeID, UseShareMemory>::vector source_boundary;
typename util::ShM<NodeID, UseShareMemory>::vector destination_boundary;
typename util::ShM<CellData, UseShareMemory>::vector cells;
typename util::ShM<std::size_t, UseShareMemory>::vector level_to_cell_offset;
Vector<EdgeWeight> weights;
Vector<NodeID> source_boundary;
Vector<NodeID> destination_boundary;
Vector<CellData> cells;
Vector<std::uint64_t> level_to_cell_offset;
};
}
}
}
#endif

6
src/partition/partitioner.cpp
View File

@ -252,17 +252,17 @@ int Partitioner::Run(const PartitionConfig &config)
TIMER_START(packed_mlp);
osrm::util::MultiLevelPartition mlp{partitions, level_to_num_cells};
TIMER_STOP(packed_mlp);
util::Log() << "PackedMultiLevelPartition constructed in " << TIMER_SEC(packed_mlp)
util::Log() << "MultiLevelPartition constructed in " << TIMER_SEC(packed_mlp)
<< " seconds";
TIMER_START(cell_storage);
osrm::util::CellStorage<false> storage(mlp, *edge_based_graph);
osrm::util::CellStorage storage(mlp, *edge_based_graph);
TIMER_STOP(cell_storage);
util::Log() << "CellStorage constructed in " << TIMER_SEC(cell_storage) << " seconds";
TIMER_START(writing_mld_data);
io::write(config.mld_partition_path, mlp);
storage.Write(config.mld_storage_path);
io::write(config.mld_storage_path, storage);
TIMER_STOP(writing_mld_data);
util::Log() << "MLD data writing took " << TIMER_SEC(writing_mld_data) << " seconds";
}

52
src/storage/storage.cpp
View File

@ -422,12 +422,29 @@ void Storage::PopulateLayout(DataLayout &layout)
if (boost::filesystem::exists(config.mld_storage_path))
{
auto mld_cell_storage_size =
util::CellStorage<true>().GetRequiredMemorySize(config.mld_storage_path);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_STORAGE, mld_cell_storage_size);
io::FileReader reader(config.mld_storage_path, io::FileReader::VerifyFingerprint);
const auto weights_count = reader.ReadVectorSize<EdgeWeight>();
layout.SetBlockSize<EdgeWeight>(DataLayout::MLD_CELL_WEIGHTS, weights_count);
const auto source_node_count = reader.ReadVectorSize<NodeID>();
layout.SetBlockSize<NodeID>(DataLayout::MLD_CELL_SOURCE_BOUNDARY, source_node_count);
const auto destination_node_count = reader.ReadVectorSize<NodeID>();
layout.SetBlockSize<NodeID>(DataLayout::MLD_CELL_DESTINATION_BOUNDARY,
destination_node_count);
const auto cell_count = reader.ReadVectorSize<util::CellStorage::CellData>();
layout.SetBlockSize<util::CellStorage::CellData>(DataLayout::MLD_CELLS, cell_count);
const auto level_offsets_count = reader.ReadVectorSize<std::uint64_t>();
layout.SetBlockSize<std::uint64_t>(DataLayout::MLD_CELL_LEVEL_OFFSETS,
level_offsets_count);
}
else
layout.SetBlockSize<char>(DataLayout::MLD_CELL_STORAGE, 0);
{
layout.SetBlockSize<char>(DataLayout::MLD_CELL_WEIGHTS, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_SOURCE_BOUNDARY, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_DESTINATION_BOUNDARY, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELLS, 0);
layout.SetBlockSize<char>(DataLayout::MLD_CELL_LEVEL_OFFSETS, 0);
}
}
}
@ -897,14 +914,29 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
layout.GetBlockEntries(DataLayout::MLD_CELL_TO_CHILDREN));
}
const auto mld_cell_storage_ptr =
layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::MLD_CELL_STORAGE);
const auto mld_cell_storage_size = layout.GetBlockSize(DataLayout::MLD_CELL_STORAGE);
if (boost::filesystem::exists(config.mld_storage_path))
{
util::CellStorage<true>().Read(config.mld_storage_path,
mld_cell_storage_ptr,
mld_cell_storage_ptr + mld_cell_storage_size);
io::FileReader reader(config.mld_storage_path, io::FileReader::VerifyFingerprint);
auto mld_cell_weights_ptr =
layout.GetBlockPtr<EdgeWeight, true>(memory_ptr, DataLayout::MLD_CELL_WEIGHTS);
auto mld_source_boundary_ptr =
layout.GetBlockPtr<NodeID, true>(memory_ptr, DataLayout::MLD_CELL_SOURCE_BOUNDARY);
auto mld_destination_boundary_ptr = layout.GetBlockPtr<NodeID, true>(
memory_ptr, DataLayout::MLD_CELL_DESTINATION_BOUNDARY);
auto mld_cells_ptr = layout.GetBlockPtr<util::CellStorage::CellData, true>(
memory_ptr, DataLayout::MLD_CELLS);
auto mld_cell_level_offsets_ptr = layout.GetBlockPtr<std::uint64_t, true>(
memory_ptr, DataLayout::MLD_CELL_LEVEL_OFFSETS);
reader.ReadInto(mld_cell_weights_ptr,
layout.GetBlockEntries(DataLayout::MLD_CELL_WEIGHTS));
reader.ReadInto(mld_source_boundary_ptr,
layout.GetBlockEntries(DataLayout::MLD_CELL_SOURCE_BOUNDARY));
reader.ReadInto(mld_destination_boundary_ptr,
layout.GetBlockEntries(DataLayout::MLD_CELL_DESTINATION_BOUNDARY));
reader.ReadInto(mld_cells_ptr, layout.GetBlockEntries(DataLayout::MLD_CELLS));
reader.ReadInto(mld_cell_level_offsets_ptr,
layout.GetBlockEntries(DataLayout::MLD_CELL_LEVEL_OFFSETS));
}
}
}

2
src/storage/storage_config.cpp
View File

@ -20,7 +20,7 @@ StorageConfig::StorageConfig(const boost::filesystem::path &base)
names_data_path{base.string() + ".names"}, properties_path{base.string() + ".properties"},
intersection_class_path{base.string() + ".icd"}, turn_lane_data_path{base.string() + ".tld"},
turn_lane_description_path{base.string() + ".tls"},
mld_partition_path{base.string() + ".partition"}, mld_storage_path{base.string() + ".cell"}
mld_partition_path{base.string() + ".partition"}, mld_storage_path{base.string() + ".cells"}
{
}

4
unit_tests/util/cell_storage.cpp
View File

@ -81,7 +81,7 @@ BOOST_AUTO_TEST_CASE(mutable_cell_storage)
auto graph = makeGraph(edges);
// test non-const storage
CellStorage<false> storage(mlp, graph);
CellStorage storage(mlp, graph);
// Level 1
auto cell_1_0 = storage.GetCell(1, 0);
@ -224,7 +224,7 @@ BOOST_AUTO_TEST_CASE(immutable_cell_storage)
// 3/0: 3 : 1,0,1
// test const storage
const CellStorage<false> const_storage(mlp, graph);
const CellStorage const_storage(mlp, graph);
auto const_cell_1_0 = const_storage.GetCell(1, 0);
auto const_cell_1_1 = const_storage.GetCell(1, 1);