Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Load data directly into MLPView

Browse Source
This commit is contained in:
Patrick Niklaus 2017-04-02 16:35:28 +00:00 committed by Patrick Niklaus
parent 99a87b4c83
commit c87ce2dede
6 changed files with 82 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View File

@ -909,7 +909,7 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_TO_CHILDREN) > 0);
auto level_data =
*data_layout.GetBlockPtr<partition::MultiLevelPartitionView::LevelData>(
data_layout.GetBlockPtr<partition::MultiLevelPartitionView::LevelData>(
memory_block, storage::DataLayout::MLD_LEVEL_DATA);
auto mld_partition_ptr = data_layout.GetBlockPtr<PartitionID>(

8
include/extractor/files.hpp
View File

@ -50,9 +50,9 @@ inline void writeDatasources(const boost::filesystem::path &path, Datasources &s
}
// reads .osrm.geometry
template <bool UseShareMemory>
template <storage::Ownership Ownership>
inline void readSegmentData(const boost::filesystem::path &path,
detail::SegmentDataContainerImpl<UseShareMemory> &segment_data)
detail::SegmentDataContainerImpl<Ownership> &segment_data)
{
const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
storage::io::FileReader reader{path, fingerprint};
@ -61,9 +61,9 @@ inline void readSegmentData(const boost::filesystem::path &path,
}
// writes .osrm.geometry
template <bool UseShareMemory>
template <storage::Ownership Ownership>
inline void writeSegmentData(const boost::filesystem::path &path,
const detail::SegmentDataContainerImpl<UseShareMemory> &segment_data)
const detail::SegmentDataContainerImpl<Ownership> &segment_data)
{
const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
storage::io::FileWriter writer{path, fingerprint};

12
include/partition/files.hpp
View File

@ -35,7 +35,8 @@ inline void writeGraph(const boost::filesystem::path &path,
}
// read .osrm.partition file
inline void readPartition(const boost::filesystem::path &path, MultiLevelPartition &mlp)
template<storage::Ownership Ownership>
inline void readPartition(const boost::filesystem::path &path, detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
@ -44,7 +45,8 @@ inline void readPartition(const boost::filesystem::path &path, MultiLevelPartiti
}
// writes .osrm.partition file
inline void writePartition(const boost::filesystem::path &path, const MultiLevelPartition &mlp)
template<storage::Ownership Ownership>
inline void writePartition(const boost::filesystem::path &path, const detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
@ -53,7 +55,8 @@ inline void writePartition(const boost::filesystem::path &path, const MultiLevel
}
// reads .osrm.cells file
inline void readCells(const boost::filesystem::path &path, CellStorage &storage)
template<storage::Ownership Ownership>
inline void readCells(const boost::filesystem::path &path, detail::CellStorageImpl<Ownership> &storage)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
@ -62,7 +65,8 @@ inline void readCells(const boost::filesystem::path &path, CellStorage &storage)
}
// writes .osrm.cells file
inline void writeCells(const boost::filesystem::path &path, const CellStorage &storage)
template<storage::Ownership Ownership>
inline void writeCells(const boost::filesystem::path &path, const detail::CellStorageImpl<Ownership> &storage)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};

43
include/partition/multi_level_partition.hpp
View File

@ -69,8 +69,11 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
std::array<LevelID, NUM_PARTITION_BITS> bit_to_level;
std::array<std::uint32_t, MAX_NUM_LEVEL - 1> lidx_to_children_offsets;
};
using LevelDataPtr = typename std::conditional<Ownership == storage::Ownership::View,
LevelData *,
std::unique_ptr<LevelData>>::type;
MultiLevelPartitionImpl() = default;
MultiLevelPartitionImpl();
// cell_sizes is index by level (starting at 0, the base graph).
// However level 0 always needs to have cell size 1, since it is the
@ -84,7 +87,7 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
}
template <typename = typename std::enable_if<Ownership == storage::Ownership::View>>
MultiLevelPartitionImpl(LevelData level_data,
MultiLevelPartitionImpl(LevelDataPtr level_data,
Vector<PartitionID> partition_,
Vector<CellID> cell_to_children_)
: level_data(std::move(level_data)), partition(std::move(partition_)),
@ -97,8 +100,8 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
{
auto p = partition[node];
auto lidx = LevelIDToIndex(l);
auto masked = p & level_data.lidx_to_mask[lidx];
return masked >> level_data.lidx_to_offset[lidx];
auto masked = p & level_data->lidx_to_mask[lidx];
return masked >> level_data->lidx_to_offset[lidx];
}
LevelID GetQueryLevel(NodeID start, NodeID target, NodeID node) const
@ -113,10 +116,10 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
return 0;
auto msb = util::msb(partition[first] ^ partition[second]);
return level_data.bit_to_level[msb];
return level_data->bit_to_level[msb];
}
std::uint8_t GetNumberOfLevels() const { return level_data.num_level; }
std::uint8_t GetNumberOfLevels() const { return level_data->num_level; }
std::uint32_t GetNumberOfCells(LevelID level) const
{
@ -128,7 +131,7 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
{
BOOST_ASSERT(level > 1);
auto lidx = LevelIDToIndex(level);
auto offset = level_data.lidx_to_children_offsets[lidx];
auto offset = level_data->lidx_to_children_offsets[lidx];
return cell_to_children[offset + cell];
}
@ -137,7 +140,7 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
{
BOOST_ASSERT(level > 1);
auto lidx = LevelIDToIndex(level);
auto offset = level_data.lidx_to_children_offsets[lidx];
auto offset = level_data->lidx_to_children_offsets[lidx];
return cell_to_children[offset + cell + 1];
}
@ -153,7 +156,7 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
auto offsets = MakeLevelOffsets(lidx_to_num_cells);
auto masks = MakeLevelMasks(offsets, num_level);
auto bits = MakeBitToLevel(offsets, num_level);
return LevelData{num_level, offsets, masks, bits, {0}};
return std::make_unique<LevelData>(LevelData{num_level, offsets, masks, bits, {0}});
}
inline std::size_t LevelIDToIndex(LevelID l) const { return l - 1; }
@ -167,8 +170,8 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
{
auto lidx = LevelIDToIndex(l);
auto shifted_id = cell_id << level_data.lidx_to_offset[lidx];
auto cleared_cell = partition[node] & ~level_data.lidx_to_mask[lidx];
auto shifted_id = cell_id << level_data->lidx_to_offset[lidx];
auto cleared_cell = partition[node] & ~level_data->lidx_to_mask[lidx];
partition[node] = cleared_cell | shifted_id;
}
@ -296,13 +299,13 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
level--;
}
level_data.lidx_to_children_offsets[0] = 0;
level_data->lidx_to_children_offsets[0] = 0;
for (auto level_idx = 0UL; level_idx < partitions.size() - 1; ++level_idx)
{
const auto &parent_partition = partitions[level_idx + 1];
level_data.lidx_to_children_offsets[level_idx + 1] = cell_to_children.size();
level_data->lidx_to_children_offsets[level_idx + 1] = cell_to_children.size();
CellID last_parent_id = parent_partition[permutation.front()];
cell_to_children.push_back(GetCell(level_idx + 1, permutation.front()));
@ -321,11 +324,21 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
}
}
//! this is always owned by this class because it is so small
LevelData level_data;
LevelDataPtr level_data = {};
Vector<PartitionID> partition;
Vector<CellID> cell_to_children;
};
template <>
inline MultiLevelPartitionImpl<storage::Ownership::Container>::MultiLevelPartitionImpl()
: level_data(std::make_unique<LevelData>())
{
}
template <>
inline MultiLevelPartitionImpl<storage::Ownership::View>::MultiLevelPartitionImpl() : level_data(nullptr)
{
}
}
}
}

21
include/partition/serialization.hpp
View File

@ -17,8 +17,7 @@ namespace serialization
{
template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader,
MultiLevelGraph<EdgeDataT, Ownership> &graph)
inline void read(storage::io::FileReader &reader, MultiLevelGraph<EdgeDataT, Ownership> &graph)
{
reader.DeserializeVector(graph.node_array);
reader.DeserializeVector(graph.edge_array);
@ -34,21 +33,25 @@ inline void write(storage::io::FileWriter &writer,
writer.SerializeVector(graph.node_to_edge_offset);
}
template <> inline void read(storage::io::FileReader &reader, MultiLevelPartition &mlp)
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
reader.ReadInto<MultiLevelPartition::LevelData>(mlp.level_data);
reader.ReadInto(*mlp.level_data);
reader.DeserializeVector(mlp.partition);
reader.DeserializeVector(mlp.cell_to_children);
}
template <> inline void write(storage::io::FileWriter &writer, const MultiLevelPartition &mlp)
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
writer.WriteOne(mlp.level_data);
writer.WriteOne(*mlp.level_data);
writer.SerializeVector(mlp.partition);
writer.SerializeVector(mlp.cell_to_children);
}
template <> inline void read(storage::io::FileReader &reader, CellStorage &storage)
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage)
{
reader.DeserializeVector(storage.weights);
reader.DeserializeVector(storage.source_boundary);
@ -57,7 +60,9 @@ template <> inline void read(storage::io::FileReader &reader, CellStorage &stora
reader.DeserializeVector(storage.level_to_cell_offset);
}
template <> inline void write(storage::io::FileWriter &writer, const CellStorage &storage)
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::CellStorageImpl<Ownership> &storage)
{
writer.SerializeVector(storage.weights);
writer.SerializeVector(storage.source_boundary);

53
src/storage/storage.cpp
View File

@ -11,6 +11,7 @@
#include "extractor/travel_mode.hpp"
#include "partition/cell_storage.hpp"
#include "partition/edge_based_graph_reader.hpp"
#include "partition/files.hpp"
#include "partition/multi_level_partition.hpp"
#include "storage/io.hpp"
#include "storage/serialization.hpp"
@ -653,8 +654,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
util::vector_view<unsigned> geometry_begin_indices(
geometries_index_ptr, layout.num_entries[storage::DataLayout::GEOMETRIES_INDEX]);
auto geometries_node_list_ptr = layout.GetBlockPtr<NodeID, true>(
memory_ptr, storage::DataLayout::GEOMETRIES_NODE_LIST);
auto geometries_node_list_ptr =
layout.GetBlockPtr<NodeID, true>(memory_ptr, storage::DataLayout::GEOMETRIES_NODE_LIST);
util::vector_view<NodeID> geometry_node_list(
geometries_node_list_ptr,
layout.num_entries[storage::DataLayout::GEOMETRIES_NODE_LIST]);
@ -689,12 +690,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
datasources_list_ptr, layout.num_entries[storage::DataLayout::DATASOURCES_LIST]);
extractor::SegmentDataView segment_data{std::move(geometry_begin_indices),
std::move(geometry_node_list),
std::move(geometry_fwd_weight_list),
std::move(geometry_rev_weight_list),
std::move(geometry_fwd_duration_list),
std::move(geometry_rev_duration_list),
std::move(datasources_list)};
std::move(geometry_node_list),
std::move(geometry_fwd_weight_list),
std::move(geometry_rev_weight_list),
std::move(geometry_fwd_duration_list),
std::move(geometry_rev_duration_list),
std::move(datasources_list)};
extractor::files::readSegmentData(config.geometries_path, segment_data);
}
@ -898,27 +899,29 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
// Loading MLD Data
if (boost::filesystem::exists(config.mld_partition_path))
{
auto mld_level_data_ptr =
layout.GetBlockPtr<partition::MultiLevelPartition::LevelData, true>(
memory_ptr, DataLayout::MLD_LEVEL_DATA);
auto mld_partition_ptr =
layout.GetBlockPtr<PartitionID, true>(memory_ptr, DataLayout::MLD_PARTITION);
auto mld_chilren_ptr =
layout.GetBlockPtr<CellID, true>(memory_ptr, DataLayout::MLD_CELL_TO_CHILDREN);
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0);
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELL_TO_CHILDREN) > 0);
BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_PARTITION) > 0);
io::FileReader reader(config.mld_partition_path, io::FileReader::VerifyFingerprint);
auto level_data =
layout.GetBlockPtr<partition::MultiLevelPartitionView::LevelData, true>(
memory_ptr, storage::DataLayout::MLD_LEVEL_DATA);
reader.ReadInto(mld_level_data_ptr, 1);
auto mld_partition_ptr = layout.GetBlockPtr<PartitionID, true>(
memory_ptr, storage::DataLayout::MLD_PARTITION);
auto partition_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_PARTITION);
util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count);
std::uint64_t size;
auto mld_chilren_ptr = layout.GetBlockPtr<CellID, true>(
memory_ptr, storage::DataLayout::MLD_CELL_TO_CHILDREN);
auto children_entries_count =
layout.GetBlockEntries(storage::DataLayout::MLD_CELL_TO_CHILDREN);
util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count);
reader.ReadInto(size);
BOOST_ASSERT(size == layout.GetBlockEntries(DataLayout::MLD_PARTITION));
reader.ReadInto(mld_partition_ptr, size);
reader.ReadInto(size);
BOOST_ASSERT(size == layout.GetBlockEntries(DataLayout::MLD_CELL_TO_CHILDREN));
reader.ReadInto(mld_chilren_ptr, size);
partition::MultiLevelPartitionView mlp{
std::move(level_data), std::move(partition), std::move(cell_to_children)};
partition::files::readPartition(config.mld_partition_path, mlp);
}
if (boost::filesystem::exists(config.mld_storage_path))