Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Load data directly into MLPView

Browse Source
This commit is contained in:
Patrick Niklaus
2017-04-02 16:35:28 +00:00
committed by Patrick Niklaus
parent 99a87b4c83
commit c87ce2dede
6 changed files with 82 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+1 -1
View File
@@ -909,7 +909,7 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_TO_CHILDREN) > 0);
auto level_data =
*data_layout.GetBlockPtr<partition::MultiLevelPartitionView::LevelData>(
data_layout.GetBlockPtr<partition::MultiLevelPartitionView::LevelData>(
memory_block, storage::DataLayout::MLD_LEVEL_DATA);
auto mld_partition_ptr = data_layout.GetBlockPtr<PartitionID>(
include/extractor/files.hpp
+4 -4
View File
@@ -50,9 +50,9 @@ inline void writeDatasources(const boost::filesystem::path &path, Datasources &s
}
// reads .osrm.geometry
template <bool UseShareMemory>
template <storage::Ownership Ownership>
inline void readSegmentData(const boost::filesystem::path &path,
detail::SegmentDataContainerImpl<UseShareMemory> &segment_data)
detail::SegmentDataContainerImpl<Ownership> &segment_data)
{
const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
storage::io::FileReader reader{path, fingerprint};
@@ -61,9 +61,9 @@ inline void readSegmentData(const boost::filesystem::path &path,
}
// writes .osrm.geometry
template <bool UseShareMemory>
template <storage::Ownership Ownership>
inline void writeSegmentData(const boost::filesystem::path &path,
const detail::SegmentDataContainerImpl<UseShareMemory> &segment_data)
const detail::SegmentDataContainerImpl<Ownership> &segment_data)
{
const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
storage::io::FileWriter writer{path, fingerprint};
include/partition/files.hpp
+8 -4
View File
@@ -35,7 +35,8 @@ inline void writeGraph(const boost::filesystem::path &path,
}
// read .osrm.partition file
inline void readPartition(const boost::filesystem::path &path, MultiLevelPartition &mlp)
template<storage::Ownership Ownership>
inline void readPartition(const boost::filesystem::path &path, detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
@@ -44,7 +45,8 @@ inline void readPartition(const boost::filesystem::path &path, MultiLevelPartiti
}
// writes .osrm.partition file
inline void writePartition(const boost::filesystem::path &path, const MultiLevelPartition &mlp)
template<storage::Ownership Ownership>
inline void writePartition(const boost::filesystem::path &path, const detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
@@ -53,7 +55,8 @@ inline void writePartition(const boost::filesystem::path &path, const MultiLevel
}
// reads .osrm.cells file
inline void readCells(const boost::filesystem::path &path, CellStorage &storage)
template<storage::Ownership Ownership>
inline void readCells(const boost::filesystem::path &path, detail::CellStorageImpl<Ownership> &storage)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
@@ -62,7 +65,8 @@ inline void readCells(const boost::filesystem::path &path, CellStorage &storage)
}
// writes .osrm.cells file
inline void writeCells(const boost::filesystem::path &path, const CellStorage &storage)
template<storage::Ownership Ownership>
inline void writeCells(const boost::filesystem::path &path, const detail::CellStorageImpl<Ownership> &storage)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
include/partition/multi_level_partition.hpp
+28 -15
View File
@@ -69,8 +69,11 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
std::array<LevelID, NUM_PARTITION_BITS> bit_to_level;
std::array<std::uint32_t, MAX_NUM_LEVEL - 1> lidx_to_children_offsets;
};
using LevelDataPtr = typename std::conditional<Ownership == storage::Ownership::View,
LevelData *,
std::unique_ptr<LevelData>>::type;
MultiLevelPartitionImpl() = default;
MultiLevelPartitionImpl();
// cell_sizes is index by level (starting at 0, the base graph).
// However level 0 always needs to have cell size 1, since it is the
@@ -84,7 +87,7 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
}
template <typename = typename std::enable_if<Ownership == storage::Ownership::View>>
MultiLevelPartitionImpl(LevelData level_data,
MultiLevelPartitionImpl(LevelDataPtr level_data,
Vector<PartitionID> partition_,
Vector<CellID> cell_to_children_)
: level_data(std::move(level_data)), partition(std::move(partition_)),
@@ -97,8 +100,8 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
{
auto p = partition[node];
auto lidx = LevelIDToIndex(l);
auto masked = p & level_data.lidx_to_mask[lidx];
return masked >> level_data.lidx_to_offset[lidx];
auto masked = p & level_data->lidx_to_mask[lidx];
return masked >> level_data->lidx_to_offset[lidx];
}
LevelID GetQueryLevel(NodeID start, NodeID target, NodeID node) const
@@ -113,10 +116,10 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
return 0;
auto msb = util::msb(partition[first] ^ partition[second]);
return level_data.bit_to_level[msb];
return level_data->bit_to_level[msb];
}
std::uint8_t GetNumberOfLevels() const { return level_data.num_level; }
std::uint8_t GetNumberOfLevels() const { return level_data->num_level; }
std::uint32_t GetNumberOfCells(LevelID level) const
{
@@ -128,7 +131,7 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
{
BOOST_ASSERT(level > 1);
auto lidx = LevelIDToIndex(level);
auto offset = level_data.lidx_to_children_offsets[lidx];
auto offset = level_data->lidx_to_children_offsets[lidx];
return cell_to_children[offset + cell];
}
@@ -137,7 +140,7 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
{
BOOST_ASSERT(level > 1);
auto lidx = LevelIDToIndex(level);
auto offset = level_data.lidx_to_children_offsets[lidx];
auto offset = level_data->lidx_to_children_offsets[lidx];
return cell_to_children[offset + cell + 1];
}
@@ -153,7 +156,7 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
auto offsets = MakeLevelOffsets(lidx_to_num_cells);
auto masks = MakeLevelMasks(offsets, num_level);
auto bits = MakeBitToLevel(offsets, num_level);
return LevelData{num_level, offsets, masks, bits, {0}};
return std::make_unique<LevelData>(LevelData{num_level, offsets, masks, bits, {0}});
}
inline std::size_t LevelIDToIndex(LevelID l) const { return l - 1; }
@@ -167,8 +170,8 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
{
auto lidx = LevelIDToIndex(l);
auto shifted_id = cell_id << level_data.lidx_to_offset[lidx];
auto cleared_cell = partition[node] & ~level_data.lidx_to_mask[lidx];
auto shifted_id = cell_id << level_data->lidx_to_offset[lidx];
auto cleared_cell = partition[node] & ~level_data->lidx_to_mask[lidx];
partition[node] = cleared_cell | shifted_id;
}
@@ -296,13 +299,13 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
level--;
}
level_data.lidx_to_children_offsets[0] = 0;
level_data->lidx_to_children_offsets[0] = 0;
for (auto level_idx = 0UL; level_idx < partitions.size() - 1; ++level_idx)
{
const auto &parent_partition = partitions[level_idx + 1];
level_data.lidx_to_children_offsets[level_idx + 1] = cell_to_children.size();
level_data->lidx_to_children_offsets[level_idx + 1] = cell_to_children.size();
CellID last_parent_id = parent_partition[permutation.front()];
cell_to_children.push_back(GetCell(level_idx + 1, permutation.front()));
@@ -321,11 +324,21 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
}
}
//! this is always owned by this class because it is so small
LevelData level_data;
LevelDataPtr level_data = {};
Vector<PartitionID> partition;
Vector<CellID> cell_to_children;
};
template <>
inline MultiLevelPartitionImpl<storage::Ownership::Container>::MultiLevelPartitionImpl()
: level_data(std::make_unique<LevelData>())
{
}
template <>
inline MultiLevelPartitionImpl<storage::Ownership::View>::MultiLevelPartitionImpl() : level_data(nullptr)
{
}
}
}
}
include/partition/serialization.hpp
+13 -8
View File
@@ -17,8 +17,7 @@ namespace serialization
{
template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader,
MultiLevelGraph<EdgeDataT, Ownership> &graph)
inline void read(storage::io::FileReader &reader, MultiLevelGraph<EdgeDataT, Ownership> &graph)
{
reader.DeserializeVector(graph.node_array);
reader.DeserializeVector(graph.edge_array);
@@ -34,21 +33,25 @@ inline void write(storage::io::FileWriter &writer,
writer.SerializeVector(graph.node_to_edge_offset);
}
template <> inline void read(storage::io::FileReader &reader, MultiLevelPartition &mlp)
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
reader.ReadInto<MultiLevelPartition::LevelData>(mlp.level_data);
reader.ReadInto(*mlp.level_data);
reader.DeserializeVector(mlp.partition);
reader.DeserializeVector(mlp.cell_to_children);
}
template <> inline void write(storage::io::FileWriter &writer, const MultiLevelPartition &mlp)
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
writer.WriteOne(mlp.level_data);
writer.WriteOne(*mlp.level_data);
writer.SerializeVector(mlp.partition);
writer.SerializeVector(mlp.cell_to_children);
}
template <> inline void read(storage::io::FileReader &reader, CellStorage &storage)
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage)
{
reader.DeserializeVector(storage.weights);
reader.DeserializeVector(storage.source_boundary);
@@ -57,7 +60,9 @@ template <> inline void read(storage::io::FileReader &reader, CellStorage &stora
reader.DeserializeVector(storage.level_to_cell_offset);
}
template <> inline void write(storage::io::FileWriter &writer, const CellStorage &storage)
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::CellStorageImpl<Ownership> &storage)
{
writer.SerializeVector(storage.weights);
writer.SerializeVector(storage.source_boundary);