Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Initial interation on writing out MLD partition/cell data

Browse Source
This commit is contained in:
Michael Krasnyk
2017-02-23 16:39:29 +01:00
committed by Patrick Niklaus
parent 757e7ca936
commit ff0a98196f
15 changed files with 415 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+35
View File
@@ -16,10 +16,12 @@
#include "engine/algorithm.hpp"
#include "engine/geospatial_query.hpp"
#include "util/cell_storage.hpp"
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
#include "util/guidance/turn_bearing.hpp"
#include "util/log.hpp"
#include "util/multi_level_partition.hpp"
#include "util/name_table.hpp"
#include "util/packed_vector.hpp"
#include "util/range_table.hpp"
@@ -249,6 +251,10 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
std::shared_ptr<util::RangeTable<16, true>> m_bearing_ranges_table;
util::ShM<DiscreteBearing, true>::vector m_bearing_values_table;
// MLD data
util::PackedMultiLevelPartition<true> mld_partition;
util::CellStorage<true> mld_cell_storage;
// allocator that keeps the allocation data
std::shared_ptr<ContiguousBlockAllocator> allocator;
@@ -534,6 +540,27 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
m_entry_class_table = std::move(entry_class_table);
}
void InitializeMLDDataPointers(storage::DataLayout &data_layout, char *memory_block)
{
if (data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_PARTITION) > 0)
{
auto mld_partition_ptr =
data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::MLD_CELL_PARTITION);
mld_partition.InitializePointers(
mld_partition_ptr,
mld_partition_ptr + data_layout.num_entries[storage::DataLayout::MLD_CELL_PARTITION]);
}
if (data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_STORAGE) > 0)
{
auto mld_cell_storage_ptr =
data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::MLD_CELL_STORAGE);
mld_cell_storage.InitializePointers(
mld_cell_storage_ptr,
mld_cell_storage_ptr + data_layout.num_entries[storage::DataLayout::MLD_CELL_STORAGE]);
}
}
void InitializeInternalPointers(storage::DataLayout &data_layout, char *memory_block)
{
InitializeChecksumPointer(data_layout, memory_block);
@@ -547,6 +574,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
InitializeProfilePropertiesPointer(data_layout, memory_block);
InitializeRTreePointers(data_layout, memory_block);
InitializeIntersectionClassPointers(data_layout, memory_block);
InitializeMLDDataPointers(data_layout, memory_block);
}
public:
@@ -1049,6 +1077,13 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
m_lane_description_masks.begin() +
m_lane_description_offsets[lane_description_id + 1]);
}
const util::PackedMultiLevelPartition<true> &GetMultiLevelPartition() const
{
return mld_partition;
}
const util::CellStorage<true> &GetCellStorage() const { return mld_cell_storage; }
};
template <typename AlgorithmT> class ContiguousInternalMemoryDataFacade;
include/partition/partition_config.hpp
+8 -1
View File
@@ -13,7 +13,10 @@ namespace partition
struct PartitionConfig
{
PartitionConfig() : requested_num_threads(0) {}
PartitionConfig()
: requested_num_threads(0)
{
}
void UseDefaults()
{
@@ -34,6 +37,8 @@ struct PartitionConfig
compressed_node_based_graph_path = basepath + ".osrm.cnbg";
cnbg_ebg_mapping_path = basepath + ".osrm.cnbg_to_ebg";
partition_path = basepath + ".osrm.partition";
mld_partition_path = basepath + ".osrm.mld_partition";
mld_storage_path = basepath + ".osrm.mld_storage";
}
// might be changed to the node based graph at some point
@@ -42,6 +47,8 @@ struct PartitionConfig
boost::filesystem::path compressed_node_based_graph_path;
boost::filesystem::path cnbg_ebg_mapping_path;
boost::filesystem::path partition_path;
boost::filesystem::path mld_partition_path;
boost::filesystem::path mld_storage_path;
unsigned requested_num_threads;
include/partition/recursive_bisection.hpp
+2
View File
@@ -26,6 +26,8 @@ class RecursiveBisection
const std::vector<BisectionID> &BisectionIDs() const;
std::uint32_t SCCDepth() const;
private:
BisectionGraph &bisection_graph;
RecursiveBisectionState internal_state;
include/storage/io.hpp
+20
View File
@@ -134,6 +134,26 @@ class FileReader
ReadInto(data.data(), count);
}
template <typename T> std::size_t GetVectorMemorySize()
{
const auto count = ReadElementCount64();
Skip<T>(count);
return sizeof(count) + sizeof(T) * count;
}
template <typename T> void *DeserializeVector(void *begin, const void *end)
{
auto count = ReadElementCount64();
auto required = reinterpret_cast<char *>(begin) + sizeof(count) + sizeof(T) * count;
if (required > end)
throw util::exception("Not enough memory ");
*reinterpret_cast<decltype(count) *>(begin) = count;
ReadInto(reinterpret_cast<T *>(reinterpret_cast<char *>(begin) + sizeof(decltype(count))),
count);
return required;
}
bool ReadAndCheckFingerprint()
{
auto loaded_fingerprint = ReadOne<util::FingerPrint>();
include/storage/shared_datatype.hpp
+5 -1
View File
@@ -56,7 +56,9 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA",
"LANE_DESCRIPTION_OFFSETS",
"LANE_DESCRIPTION_MASKS",
"TURN_WEIGHT_PENALTIES",
"TURN_DURATION_PENALTIES"};
"TURN_DURATION_PENALTIES",
"MLD_CELL_PARTITION",
"MLD_CELL_STORAGE"};
struct DataLayout
{
@@ -101,6 +103,8 @@ struct DataLayout
LANE_DESCRIPTION_MASKS,
TURN_WEIGHT_PENALTIES,
TURN_DURATION_PENALTIES,
MLD_CELL_PARTITION,
MLD_CELL_STORAGE,
NUM_BLOCKS
};
include/storage/storage_config.hpp
+2
View File
@@ -69,6 +69,8 @@ struct StorageConfig final
boost::filesystem::path intersection_class_path;
boost::filesystem::path turn_lane_data_path;
boost::filesystem::path turn_lane_description_path;
boost::filesystem::path mld_partition_path;
boost::filesystem::path mld_storage_path;
};
}
}
include/util/cell_storage.hpp
+102 -6
View File
@@ -3,9 +3,13 @@
#include "util/assert.hpp"
#include "util/for_each_range.hpp"
#include "util/log.hpp"
#include "util/multi_level_partition.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/typedefs.hpp"
#include "storage/io.hpp"
#include <boost/range/iterator_range.hpp>
#include <tbb/parallel_sort.h>
@@ -19,7 +23,7 @@ namespace osrm
namespace util
{
class CellStorage
template <bool UseShareMemory> class CellStorage
{
public:
using WeightOffset = std::uint32_t;
@@ -161,6 +165,8 @@ class CellStorage
using Cell = CellImpl<EdgeWeight>;
using ConstCell = CellImpl<const EdgeWeight>;
CellStorage() {}
template <typename GraphT>
CellStorage(const MultiLevelPartition &partition, const GraphT &base_graph)
{
@@ -286,6 +292,8 @@ class CellStorage
{
}
CellStorage(const boost::filesystem::path &path) { Read(path); }
ConstCell GetCell(LevelID level, CellID id) const
{
const auto level_index = LevelIDToIndex(level);
@@ -308,12 +316,100 @@ class CellStorage
cells[cell_index], weights.data(), source_boundary.data(), destination_boundary.data()};
}
std::size_t GetRequiredMemorySize(const boost::filesystem::path &path) const
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
std::size_t memory_size = 0;
memory_size += reader.GetVectorMemorySize<decltype(weights[0])>();
memory_size += reader.GetVectorMemorySize<decltype(source_boundary[0])>();
memory_size += reader.GetVectorMemorySize<decltype(destination_boundary[0])>();
memory_size += reader.GetVectorMemorySize<decltype(cells[0])>();
memory_size += reader.GetVectorMemorySize<decltype(level_to_cell_offset[0])>();
return memory_size;
}
template <bool Q = UseShareMemory>
typename std::enable_if<Q>::type
Read(const boost::filesystem::path &path, void *begin, const void *end) const
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
begin = reader.DeserializeVector<typename decltype(weights)::value_type>(begin, end);
begin =
reader.DeserializeVector<typename decltype(source_boundary)::value_type>(begin, end);
begin = reader.DeserializeVector<typename decltype(destination_boundary)::value_type>(begin,
end);
begin = reader.DeserializeVector<typename decltype(cells)::value_type>(begin, end);
begin = reader.DeserializeVector<typename decltype(level_to_cell_offset)::value_type>(begin,
end);
}
template <bool Q = UseShareMemory>
typename std::enable_if<Q>::type InitializePointers(char *begin, const char *end)
{
auto weights_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(weights_size);
weights.reset(reinterpret_cast<EdgeWeight *>(begin), weights_size);
begin += sizeof(decltype(weights[0])) * weights_size;
auto source_boundary_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(source_boundary_size);
source_boundary.reset(reinterpret_cast<NodeID *>(begin), source_boundary_size);
begin += sizeof(decltype(source_boundary[0])) * source_boundary_size;
auto destination_boundary_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(destination_boundary_size);
destination_boundary.reset(reinterpret_cast<NodeID *>(begin), destination_boundary_size);
begin += sizeof(decltype(destination_boundary[0])) * destination_boundary_size;
auto cells_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(cells_size);
cells.reset(reinterpret_cast<CellData *>(begin), cells_size);
begin += sizeof(decltype(cells[0])) * cells_size;
auto level_to_cell_offset_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(level_to_cell_offset_size);
level_to_cell_offset.reset(reinterpret_cast<std::size_t *>(begin),
level_to_cell_offset_size);
begin += sizeof(decltype(level_to_cell_offset[0])) * level_to_cell_offset_size;
BOOST_ASSERT(begin <= end);
}
template <bool Q = UseShareMemory>
typename std::enable_if<!Q>::type Read(const boost::filesystem::path &path)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(weights);
reader.DeserializeVector(source_boundary);
reader.DeserializeVector(destination_boundary);
reader.DeserializeVector(cells);
reader.DeserializeVector(level_to_cell_offset);
}
void Write(const boost::filesystem::path &path) const
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(weights);
writer.SerializeVector(source_boundary);
writer.SerializeVector(destination_boundary);
writer.SerializeVector(cells);
writer.SerializeVector(level_to_cell_offset);
}
private:
std::vector<EdgeWeight> weights;
std::vector<NodeID> source_boundary;
std::vector<NodeID> destination_boundary;
std::vector<CellData> cells;
std::vector<std::size_t> level_to_cell_offset;
typename util::ShM<EdgeWeight, UseShareMemory>::vector weights;
typename util::ShM<NodeID, UseShareMemory>::vector source_boundary;
typename util::ShM<NodeID, UseShareMemory>::vector destination_boundary;
typename util::ShM<CellData, UseShareMemory>::vector cells;
typename util::ShM<std::size_t, UseShareMemory>::vector level_to_cell_offset;
};
}
}
include/util/multi_level_partition.hpp
+95 -5
View File
@@ -3,8 +3,11 @@
#include "util/exception.hpp"
#include "util/for_each_pair.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/typedefs.hpp"
#include "storage/io.hpp"
#include <algorithm>
#include <array>
#include <climits>
@@ -73,12 +76,14 @@ class MultiLevelPartition
virtual std::uint32_t GetNumberOfCells(LevelID level) const = 0;
};
class PackedMultiLevelPartition final : public MultiLevelPartition
template <bool UseShareMemory> class PackedMultiLevelPartition final : public MultiLevelPartition
{
using PartitionID = std::uint64_t;
static const constexpr std::uint8_t NUM_PARTITION_BITS = sizeof(PartitionID) * CHAR_BIT;
public:
PackedMultiLevelPartition() {}
// cell_sizes is index by level (starting at 0, the base graph).
// However level 0 always needs to have cell size 1, since it is the
// basegraph.
@@ -90,6 +95,8 @@ class PackedMultiLevelPartition final : public MultiLevelPartition
initializePartitionIDs(partitions);
}
PackedMultiLevelPartition(const boost::filesystem::path &file_name) { Read(file_name); }
// returns the index of the cell the vertex is contained at level l
CellID GetCell(LevelID l, NodeID node) const final override
{
@@ -139,6 +146,89 @@ class PackedMultiLevelPartition final : public MultiLevelPartition
return cell_to_children[offset + cell + 1];
}
std::size_t GetRequiredMemorySize(const boost::filesystem::path &path) const
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
std::size_t memory_size = 0;
memory_size += reader.GetVectorMemorySize<decltype(level_offsets[0])>();
memory_size += reader.GetVectorMemorySize<decltype(partition[0])>();
memory_size += reader.GetVectorMemorySize<decltype(level_to_children_offset[0])>();
memory_size += reader.GetVectorMemorySize<decltype(cell_to_children[0])>();
return memory_size;
}
template <bool Q = UseShareMemory>
typename std::enable_if<Q>::type
Read(const boost::filesystem::path &path, void *begin, const void *end) const
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
begin = reader.DeserializeVector<typename decltype(level_offsets)::value_type>(begin, end);
begin = reader.DeserializeVector<typename decltype(partition)::value_type>(begin, end);
begin = reader.DeserializeVector<typename decltype(level_to_children_offset)::value_type>(
begin, end);
begin =
reader.DeserializeVector<typename decltype(cell_to_children)::value_type>(begin, end);
}
template <bool Q = UseShareMemory>
typename std::enable_if<Q>::type InitializePointers(char *begin, const char *end)
{
auto level_offsets_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(level_offsets_size);
level_offsets.reset(begin, level_offsets_size);
begin += sizeof(decltype(level_offsets[0])) * level_offsets_size;
auto partition_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(partition_size);
partition.reset(begin, partition_size);
begin += sizeof(decltype(partition[0])) * partition_size;
auto level_to_children_offset_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(level_to_children_offset_size);
level_to_children_offset.reset(begin, level_to_children_offset_size);
begin += sizeof(decltype(level_to_children_offset[0])) * level_to_children_offset_size;
auto cell_to_children_size = *reinterpret_cast<std::uint64_t *>(begin);
begin += sizeof(cell_to_children_size);
cell_to_children.reset(begin, cell_to_children_size);
begin += sizeof(decltype(cell_to_children[0])) * level_to_children_offset_size;
BOOST_ASSERT(begin <= end);
level_masks = makeLevelMasks();
bit_to_level = makeBitToLevel();
}
template <bool Q = UseShareMemory>
typename std::enable_if<!Q>::type Read(const boost::filesystem::path &path)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(level_offsets);
reader.DeserializeVector(partition);
reader.DeserializeVector(level_to_children_offset);
reader.DeserializeVector(cell_to_children);
level_masks = makeLevelMasks();
bit_to_level = makeBitToLevel();
}
void Write(const boost::filesystem::path &path) const
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(level_offsets);
writer.SerializeVector(partition);
writer.SerializeVector(level_to_children_offset);
writer.SerializeVector(cell_to_children);
}
private:
inline std::size_t LevelIDToIndex(LevelID l) const { return l - 1; }
@@ -306,11 +396,11 @@ class PackedMultiLevelPartition final : public MultiLevelPartition
}
}
std::vector<PartitionID> partition;
std::vector<std::uint8_t> level_offsets;
typename util::ShM<std::uint8_t, UseShareMemory>::vector level_offsets;
typename util::ShM<PartitionID, UseShareMemory>::vector partition;
typename util::ShM<std::uint32_t, UseShareMemory>::vector level_to_children_offset;
typename util::ShM<CellID, UseShareMemory>::vector cell_to_children;
std::vector<PartitionID> level_masks;
std::vector<std::uint32_t> level_to_children_offset;
std::vector<CellID> cell_to_children;
std::array<LevelID, NUM_PARTITION_BITS> bit_to_level;
};
}
include/util/shared_memory_vector_wrapper.hpp
+7
View File
@@ -58,6 +58,7 @@ template <typename DataT> class SharedMemoryWrapper
std::size_t m_size;
public:
using value_type = DataT;
using iterator = ShMemIterator<DataT>;
using reverse_iterator = boost::reverse_iterator<iterator>;
@@ -71,6 +72,12 @@ template <typename DataT> class SharedMemoryWrapper
m_size = size;
}
void reset(void *ptr, std::size_t size)
{
m_ptr = reinterpret_cast<DataT *>(ptr);
m_size = size;
}
DataT &at(const std::size_t index) { return m_ptr[index]; }
const DataT &at(const std::size_t index) const { return m_ptr[index]; }