From 694bf9d8b1d7908ffc59e6763127d5d57a1cd0b6 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 4 Mar 2017 18:36:29 +0000 Subject: [PATCH] Integrate CellStorage into datafacade --- .../datafacade/algorithm_datafacade.hpp | 2 +- .../contiguous_internalmem_datafacade.hpp | 55 +++++-- include/partition/io.hpp | 15 ++ include/storage/shared_datatype.hpp | 12 +- include/util/cell_storage.hpp | 141 +++++------------- src/partition/partitioner.cpp | 6 +- src/storage/storage.cpp | 52 +++++-- src/storage/storage_config.cpp | 2 +- unit_tests/util/cell_storage.cpp | 4 +- 9 files changed, 157 insertions(+), 132 deletions(-) diff --git a/include/engine/datafacade/algorithm_datafacade.hpp b/include/engine/datafacade/algorithm_datafacade.hpp index dcd38b800..d6c4bab1e 100644 --- a/include/engine/datafacade/algorithm_datafacade.hpp +++ b/include/engine/datafacade/algorithm_datafacade.hpp @@ -66,7 +66,7 @@ template <> class AlgorithmDataFacade public: virtual const util::MultiLevelPartitionView &GetMultiLevelPartition() const = 0; - virtual const util::CellStorage &GetCellStorage() const = 0; + virtual const util::CellStorageView &GetCellStorage() const = 0; }; } } diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp index 7030c7065..5850322c5 100644 --- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp +++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp @@ -1089,7 +1089,7 @@ class ContiguousInternalMemoryAlgorithmDataFacade { // MLD data util::MultiLevelPartitionView mld_partition; - util::CellStorage mld_cell_storage; + util::CellStorageView mld_cell_storage; void InitializeInternalPointers(storage::DataLayout &data_layout, char *memory_block) { @@ -1123,14 +1123,51 @@ class ContiguousInternalMemoryAlgorithmDataFacade mld_partition = util::MultiLevelPartitionView{level_data, partition, cell_to_children}; } - if (data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_STORAGE) > 0) + if (data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_WEIGHTS) > 0) { - auto mld_cell_storage_ptr = - data_layout.GetBlockPtr(memory_block, storage::DataLayout::MLD_CELL_STORAGE); - mld_cell_storage.InitializePointers( - mld_cell_storage_ptr, - mld_cell_storage_ptr + - data_layout.num_entries[storage::DataLayout::MLD_CELL_STORAGE]); + BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY) > + 0); + BOOST_ASSERT( + data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY) > 0); + BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0); + BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0); + + auto mld_cell_weights_ptr = data_layout.GetBlockPtr( + memory_block, storage::DataLayout::MLD_CELL_WEIGHTS); + auto mld_source_boundary_ptr = data_layout.GetBlockPtr( + memory_block, storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY); + auto mld_destination_boundary_ptr = data_layout.GetBlockPtr( + memory_block, storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY); + auto mld_cells_ptr = data_layout.GetBlockPtr( + memory_block, storage::DataLayout::MLD_CELLS); + auto mld_cell_level_offsets_ptr = data_layout.GetBlockPtr( + memory_block, storage::DataLayout::MLD_CELL_LEVEL_OFFSETS); + + auto weight_entries_count = + data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_WEIGHTS); + auto source_boundary_entries_count = + data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY); + auto destination_boundary_entries_count = + data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY); + auto cells_entries_counts = data_layout.GetBlockEntries(storage::DataLayout::MLD_CELLS); + auto cell_level_offsets_entries_count = + data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS); + + util::ShM::vector weights(mld_cell_weights_ptr, weight_entries_count); + util::ShM::vector source_boundary(mld_source_boundary_ptr, + source_boundary_entries_count); + util::ShM::vector destination_boundary( + mld_destination_boundary_ptr, destination_boundary_entries_count); + util::ShM::vector cells(mld_cells_ptr, + cells_entries_counts); + util::ShM::vector level_offsets(mld_cell_level_offsets_ptr, + cell_level_offsets_entries_count); + + mld_cell_storage = util::CellStorageView{std::move(weights), + std::move(source_boundary), + std::move(destination_boundary), + std::move(cells), + std::move(level_offsets)}; } } @@ -1147,7 +1184,7 @@ class ContiguousInternalMemoryAlgorithmDataFacade const util::MultiLevelPartitionView &GetMultiLevelPartition() const { return mld_partition; } - const util::CellStorage &GetCellStorage() const { return mld_cell_storage; } + const util::CellStorageView &GetCellStorage() const { return mld_cell_storage; } }; template <> diff --git a/include/partition/io.hpp b/include/partition/io.hpp index b71283f0d..0f6ec09e2 100644 --- a/include/partition/io.hpp +++ b/include/partition/io.hpp @@ -3,6 +3,7 @@ #include "storage/io.hpp" #include "util/multi_level_partition.hpp" +#include "util/cell_storage.hpp" namespace osrm { @@ -21,6 +22,20 @@ inline void write(const boost::filesystem::path &path, const util::MultiLevelPar writer.SerializeVector(mlp.partition); writer.SerializeVector(mlp.cell_to_children); } + +template <> +inline void write(const boost::filesystem::path &path, const util::CellStorage &storage) +{ + const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; + storage::io::FileWriter writer{path, fingerprint}; + + writer.SerializeVector(storage.weights); + writer.SerializeVector(storage.source_boundary); + writer.SerializeVector(storage.destination_boundary); + writer.SerializeVector(storage.cells); + writer.SerializeVector(storage.level_to_cell_offset); +} + } } } diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp index 4e89162ce..9798e0bb0 100644 --- a/include/storage/shared_datatype.hpp +++ b/include/storage/shared_datatype.hpp @@ -60,7 +60,11 @@ const constexpr char *block_id_to_name[] = {"NAME_CHAR_DATA", "MLD_LEVEL_DATA", "MLD_PARTITION", "MLD_CELL_TO_CHILDREN", - "MLD_CELL_STORAGE"}; + "MLD_CELL_WEIGHTS", + "MLD_CELL_SOURCE_BOUNDARY", + "MLD_CELL_DESTINATION_BOUNDARY", + "MLD_CELLS", + "MLD_CELL_LEVEL_OFFSETS"}; struct DataLayout { @@ -108,7 +112,11 @@ struct DataLayout MLD_LEVEL_DATA, MLD_PARTITION, MLD_CELL_TO_CHILDREN, - MLD_CELL_STORAGE, + MLD_CELL_WEIGHTS, + MLD_CELL_SOURCE_BOUNDARY, + MLD_CELL_DESTINATION_BOUNDARY, + MLD_CELLS, + MLD_CELL_LEVEL_OFFSETS, NUM_BLOCKS }; diff --git a/include/util/cell_storage.hpp b/include/util/cell_storage.hpp index 286bf98af..36b6b4890 100644 --- a/include/util/cell_storage.hpp +++ b/include/util/cell_storage.hpp @@ -22,8 +22,24 @@ namespace osrm { namespace util { +namespace detail { +template class CellStorageImpl; +} +using CellStorage = detail::CellStorageImpl; +using CellStorageView = detail::CellStorageImpl; +} +namespace partition { +namespace io { +template +inline void write(const boost::filesystem::path &path, const util::detail::CellStorageImpl &storage); +} +} -template class CellStorage +namespace util +{ +namespace detail +{ +template class CellStorageImpl { public: using WeightOffset = std::uint32_t; @@ -35,7 +51,6 @@ template class CellStorage static constexpr auto INVALID_WEIGHT_OFFSET = std::numeric_limits::max(); static constexpr auto INVALID_BOUNDARY_OFFSET = std::numeric_limits::max(); - private: struct CellData { WeightOffset weight_offset = INVALID_WEIGHT_OFFSET; @@ -45,6 +60,9 @@ template class CellStorage BoundarySize num_destination_nodes = 0; }; + private: + template using Vector = typename util::ShM::vector; + // Implementation of the cell view. We need a template parameter here // because we need to derive a read-only and read-write view from this. template class CellImpl @@ -165,10 +183,10 @@ template class CellStorage using Cell = CellImpl; using ConstCell = CellImpl; - CellStorage() {} + CellStorageImpl() {} - template - CellStorage(const MultiLevelPartition &partition, const GraphT &base_graph) + template > + CellStorageImpl(const MultiLevelPartition &partition, const GraphT &base_graph) { // pre-allocate storge for CellData so we can have random access to it by cell id unsigned number_of_cells = 0; @@ -281,19 +299,18 @@ template class CellStorage weights.resize(weight_offset + 1, INVALID_EDGE_WEIGHT); } - CellStorage(std::vector weights_, - std::vector source_boundary_, - std::vector destination_boundary_, - std::vector cells_, - std::vector level_to_cell_offset_) + template > + CellStorageImpl(Vector weights_, + Vector source_boundary_, + Vector destination_boundary_, + Vector cells_, + Vector level_to_cell_offset_) : weights(std::move(weights_)), source_boundary(std::move(source_boundary_)), destination_boundary(std::move(destination_boundary_)), cells(std::move(cells_)), level_to_cell_offset(std::move(level_to_cell_offset_)) { } - CellStorage(const boost::filesystem::path &path) { Read(path); } - ConstCell GetCell(LevelID level, CellID id) const { const auto level_index = LevelIDToIndex(level); @@ -305,6 +322,7 @@ template class CellStorage cells[cell_index], weights.data(), source_boundary.data(), destination_boundary.data()}; } + template > Cell GetCell(LevelID level, CellID id) { const auto level_index = LevelIDToIndex(level); @@ -316,102 +334,17 @@ template class CellStorage cells[cell_index], weights.data(), source_boundary.data(), destination_boundary.data()}; } - std::size_t GetRequiredMemorySize(const boost::filesystem::path &path) const - { - const auto fingerprint = storage::io::FileReader::VerifyFingerprint; - storage::io::FileReader reader{path, fingerprint}; - - std::size_t memory_size = 0; - memory_size += reader.GetVectorMemorySize(); - memory_size += reader.GetVectorMemorySize(); - memory_size += reader.GetVectorMemorySize(); - memory_size += reader.GetVectorMemorySize(); - memory_size += reader.GetVectorMemorySize(); - return memory_size; - } - - template - typename std::enable_if::type - Read(const boost::filesystem::path &path, void *begin, const void *end) const - { - const auto fingerprint = storage::io::FileReader::VerifyFingerprint; - storage::io::FileReader reader{path, fingerprint}; - - begin = reader.DeserializeVector(begin, end); - begin = - reader.DeserializeVector(begin, end); - begin = reader.DeserializeVector(begin, - end); - begin = reader.DeserializeVector(begin, end); - begin = reader.DeserializeVector(begin, - end); - } - - template - typename std::enable_if::type InitializePointers(char *begin, const char *end) - { - auto weights_size = *reinterpret_cast(begin); - begin += sizeof(weights_size); - weights.reset(reinterpret_cast(begin), weights_size); - begin += sizeof(decltype(weights[0])) * weights_size; - - auto source_boundary_size = *reinterpret_cast(begin); - begin += sizeof(source_boundary_size); - source_boundary.reset(reinterpret_cast(begin), source_boundary_size); - begin += sizeof(decltype(source_boundary[0])) * source_boundary_size; - - auto destination_boundary_size = *reinterpret_cast(begin); - begin += sizeof(destination_boundary_size); - destination_boundary.reset(reinterpret_cast(begin), destination_boundary_size); - begin += sizeof(decltype(destination_boundary[0])) * destination_boundary_size; - - auto cells_size = *reinterpret_cast(begin); - begin += sizeof(cells_size); - cells.reset(reinterpret_cast(begin), cells_size); - begin += sizeof(decltype(cells[0])) * cells_size; - - auto level_to_cell_offset_size = *reinterpret_cast(begin); - begin += sizeof(level_to_cell_offset_size); - level_to_cell_offset.reset(reinterpret_cast(begin), - level_to_cell_offset_size); - begin += sizeof(decltype(level_to_cell_offset[0])) * level_to_cell_offset_size; - - BOOST_ASSERT(begin <= end); - } - - template - typename std::enable_if::type Read(const boost::filesystem::path &path) - { - const auto fingerprint = storage::io::FileReader::VerifyFingerprint; - storage::io::FileReader reader{path, fingerprint}; - - reader.DeserializeVector(weights); - reader.DeserializeVector(source_boundary); - reader.DeserializeVector(destination_boundary); - reader.DeserializeVector(cells); - reader.DeserializeVector(level_to_cell_offset); - } - - void Write(const boost::filesystem::path &path) const - { - const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; - storage::io::FileWriter writer{path, fingerprint}; - - writer.SerializeVector(weights); - writer.SerializeVector(source_boundary); - writer.SerializeVector(destination_boundary); - writer.SerializeVector(cells); - writer.SerializeVector(level_to_cell_offset); - } + friend void partition::io::write(const boost::filesystem::path &path, const util::detail::CellStorageImpl &storage); private: - typename util::ShM::vector weights; - typename util::ShM::vector source_boundary; - typename util::ShM::vector destination_boundary; - typename util::ShM::vector cells; - typename util::ShM::vector level_to_cell_offset; + Vector weights; + Vector source_boundary; + Vector destination_boundary; + Vector cells; + Vector level_to_cell_offset; }; } } +} #endif diff --git a/src/partition/partitioner.cpp b/src/partition/partitioner.cpp index 794fe3042..e0ca43b40 100644 --- a/src/partition/partitioner.cpp +++ b/src/partition/partitioner.cpp @@ -252,17 +252,17 @@ int Partitioner::Run(const PartitionConfig &config) TIMER_START(packed_mlp); osrm::util::MultiLevelPartition mlp{partitions, level_to_num_cells}; TIMER_STOP(packed_mlp); - util::Log() << "PackedMultiLevelPartition constructed in " << TIMER_SEC(packed_mlp) + util::Log() << "MultiLevelPartition constructed in " << TIMER_SEC(packed_mlp) << " seconds"; TIMER_START(cell_storage); - osrm::util::CellStorage storage(mlp, *edge_based_graph); + osrm::util::CellStorage storage(mlp, *edge_based_graph); TIMER_STOP(cell_storage); util::Log() << "CellStorage constructed in " << TIMER_SEC(cell_storage) << " seconds"; TIMER_START(writing_mld_data); io::write(config.mld_partition_path, mlp); - storage.Write(config.mld_storage_path); + io::write(config.mld_storage_path, storage); TIMER_STOP(writing_mld_data); util::Log() << "MLD data writing took " << TIMER_SEC(writing_mld_data) << " seconds"; } diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 34712d8f2..a76ade04e 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -422,12 +422,29 @@ void Storage::PopulateLayout(DataLayout &layout) if (boost::filesystem::exists(config.mld_storage_path)) { - auto mld_cell_storage_size = - util::CellStorage().GetRequiredMemorySize(config.mld_storage_path); - layout.SetBlockSize(DataLayout::MLD_CELL_STORAGE, mld_cell_storage_size); + io::FileReader reader(config.mld_storage_path, io::FileReader::VerifyFingerprint); + + const auto weights_count = reader.ReadVectorSize(); + layout.SetBlockSize(DataLayout::MLD_CELL_WEIGHTS, weights_count); + const auto source_node_count = reader.ReadVectorSize(); + layout.SetBlockSize(DataLayout::MLD_CELL_SOURCE_BOUNDARY, source_node_count); + const auto destination_node_count = reader.ReadVectorSize(); + layout.SetBlockSize(DataLayout::MLD_CELL_DESTINATION_BOUNDARY, + destination_node_count); + const auto cell_count = reader.ReadVectorSize(); + layout.SetBlockSize(DataLayout::MLD_CELLS, cell_count); + const auto level_offsets_count = reader.ReadVectorSize(); + layout.SetBlockSize(DataLayout::MLD_CELL_LEVEL_OFFSETS, + level_offsets_count); } else - layout.SetBlockSize(DataLayout::MLD_CELL_STORAGE, 0); + { + layout.SetBlockSize(DataLayout::MLD_CELL_WEIGHTS, 0); + layout.SetBlockSize(DataLayout::MLD_CELL_SOURCE_BOUNDARY, 0); + layout.SetBlockSize(DataLayout::MLD_CELL_DESTINATION_BOUNDARY, 0); + layout.SetBlockSize(DataLayout::MLD_CELLS, 0); + layout.SetBlockSize(DataLayout::MLD_CELL_LEVEL_OFFSETS, 0); + } } } @@ -897,14 +914,29 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) layout.GetBlockEntries(DataLayout::MLD_CELL_TO_CHILDREN)); } - const auto mld_cell_storage_ptr = - layout.GetBlockPtr(memory_ptr, DataLayout::MLD_CELL_STORAGE); - const auto mld_cell_storage_size = layout.GetBlockSize(DataLayout::MLD_CELL_STORAGE); if (boost::filesystem::exists(config.mld_storage_path)) { - util::CellStorage().Read(config.mld_storage_path, - mld_cell_storage_ptr, - mld_cell_storage_ptr + mld_cell_storage_size); + io::FileReader reader(config.mld_storage_path, io::FileReader::VerifyFingerprint); + auto mld_cell_weights_ptr = + layout.GetBlockPtr(memory_ptr, DataLayout::MLD_CELL_WEIGHTS); + auto mld_source_boundary_ptr = + layout.GetBlockPtr(memory_ptr, DataLayout::MLD_CELL_SOURCE_BOUNDARY); + auto mld_destination_boundary_ptr = layout.GetBlockPtr( + memory_ptr, DataLayout::MLD_CELL_DESTINATION_BOUNDARY); + auto mld_cells_ptr = layout.GetBlockPtr( + memory_ptr, DataLayout::MLD_CELLS); + auto mld_cell_level_offsets_ptr = layout.GetBlockPtr( + memory_ptr, DataLayout::MLD_CELL_LEVEL_OFFSETS); + + reader.ReadInto(mld_cell_weights_ptr, + layout.GetBlockEntries(DataLayout::MLD_CELL_WEIGHTS)); + reader.ReadInto(mld_source_boundary_ptr, + layout.GetBlockEntries(DataLayout::MLD_CELL_SOURCE_BOUNDARY)); + reader.ReadInto(mld_destination_boundary_ptr, + layout.GetBlockEntries(DataLayout::MLD_CELL_DESTINATION_BOUNDARY)); + reader.ReadInto(mld_cells_ptr, layout.GetBlockEntries(DataLayout::MLD_CELLS)); + reader.ReadInto(mld_cell_level_offsets_ptr, + layout.GetBlockEntries(DataLayout::MLD_CELL_LEVEL_OFFSETS)); } } } diff --git a/src/storage/storage_config.cpp b/src/storage/storage_config.cpp index eba91c7c2..2127a9ac8 100644 --- a/src/storage/storage_config.cpp +++ b/src/storage/storage_config.cpp @@ -20,7 +20,7 @@ StorageConfig::StorageConfig(const boost::filesystem::path &base) names_data_path{base.string() + ".names"}, properties_path{base.string() + ".properties"}, intersection_class_path{base.string() + ".icd"}, turn_lane_data_path{base.string() + ".tld"}, turn_lane_description_path{base.string() + ".tls"}, - mld_partition_path{base.string() + ".partition"}, mld_storage_path{base.string() + ".cell"} + mld_partition_path{base.string() + ".partition"}, mld_storage_path{base.string() + ".cells"} { } diff --git a/unit_tests/util/cell_storage.cpp b/unit_tests/util/cell_storage.cpp index 81cca0f00..437394591 100644 --- a/unit_tests/util/cell_storage.cpp +++ b/unit_tests/util/cell_storage.cpp @@ -81,7 +81,7 @@ BOOST_AUTO_TEST_CASE(mutable_cell_storage) auto graph = makeGraph(edges); // test non-const storage - CellStorage storage(mlp, graph); + CellStorage storage(mlp, graph); // Level 1 auto cell_1_0 = storage.GetCell(1, 0); @@ -224,7 +224,7 @@ BOOST_AUTO_TEST_CASE(immutable_cell_storage) // 3/0: 3 : 1,0,1 // test const storage - const CellStorage const_storage(mlp, graph); + const CellStorage const_storage(mlp, graph); auto const_cell_1_0 = const_storage.GetCell(1, 0); auto const_cell_1_1 = const_storage.GetCell(1, 1);