diff --git a/include/partitioner/cell_storage.hpp b/include/partitioner/cell_storage.hpp index 03db913ca..4f32062b7 100644 --- a/include/partitioner/cell_storage.hpp +++ b/include/partitioner/cell_storage.hpp @@ -38,9 +38,12 @@ using CellStorageView = detail::CellStorageImpl; namespace serialization { template -inline void read(storage::io::FileReader &reader, detail::CellStorageImpl &storage); +inline void read(storage::tar::FileReader &reader, + const std::string &name, + detail::CellStorageImpl &storage); template -inline void write(storage::io::FileWriter &writer, +inline void write(storage::tar::FileWriter &writer, + const std::string &name, const detail::CellStorageImpl &storage); } @@ -92,10 +95,9 @@ template class CellStorageImpl WeightValueT, boost::random_access_traversal_tag> { - typedef boost::iterator_facade - base_t; + typedef boost:: + iterator_facade + base_t; public: typedef typename base_t::value_type value_type; @@ -180,8 +182,8 @@ template class CellStorageImpl const NodeID *const all_destinations) : num_source_nodes{data.num_source_nodes}, num_destination_nodes{data.num_destination_nodes}, - weights{all_weights + data.value_offset}, - durations{all_durations + data.value_offset}, + weights{all_weights + data.value_offset}, durations{all_durations + + data.value_offset}, source_boundary{all_sources + data.source_boundary_offset}, destination_boundary{all_destinations + data.destination_boundary_offset} { @@ -390,9 +392,11 @@ template class CellStorageImpl destination_boundary.data()}; } - friend void serialization::read(storage::io::FileReader &reader, + friend void serialization::read(storage::tar::FileReader &reader, + const std::string &name, detail::CellStorageImpl &storage); - friend void serialization::write(storage::io::FileWriter &writer, + friend void serialization::write(storage::tar::FileWriter &writer, + const std::string &name, const detail::CellStorageImpl &storage); private: diff --git a/include/partitioner/files.hpp b/include/partitioner/files.hpp index f28862b08..0414ee3c8 100644 --- a/include/partitioner/files.hpp +++ b/include/partitioner/files.hpp @@ -52,10 +52,10 @@ inline void readPartition(const boost::filesystem::path &path, MultiLevelPartiti std::is_same::value, ""); - const auto fingerprint = storage::io::FileReader::VerifyFingerprint; - storage::io::FileReader reader{path, fingerprint}; + const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; + storage::tar::FileReader reader{path, fingerprint}; - serialization::read(reader, mlp); + serialization::read(reader, "/mld/multilevelpartition", mlp); } // writes .osrm.partition file @@ -66,10 +66,10 @@ inline void writePartition(const boost::filesystem::path &path, const MultiLevel std::is_same::value, ""); - const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; - storage::io::FileWriter writer{path, fingerprint}; + const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; + storage::tar::FileWriter writer{path, fingerprint}; - serialization::write(writer, mlp); + serialization::write(writer, "/mld/multilevelpartition", mlp); } // reads .osrm.cells file @@ -80,10 +80,10 @@ inline void readCells(const boost::filesystem::path &path, CellStorageT &storage std::is_same::value, ""); - const auto fingerprint = storage::io::FileReader::VerifyFingerprint; - storage::io::FileReader reader{path, fingerprint}; + const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; + storage::tar::FileReader reader{path, fingerprint}; - serialization::read(reader, storage); + serialization::read(reader, "/mld/cellstorage", storage); } // writes .osrm.cells file @@ -94,10 +94,10 @@ inline void writeCells(const boost::filesystem::path &path, CellStorageT &storag std::is_same::value, ""); - const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; - storage::io::FileWriter writer{path, fingerprint}; + const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; + storage::tar::FileWriter writer{path, fingerprint}; - serialization::write(writer, storage); + serialization::write(writer, "/mld/cellstorage", storage); } } } diff --git a/include/partitioner/multi_level_partition.hpp b/include/partitioner/multi_level_partition.hpp index 58fdd3465..aa944fc9f 100644 --- a/include/partitioner/multi_level_partition.hpp +++ b/include/partitioner/multi_level_partition.hpp @@ -7,7 +7,7 @@ #include "util/typedefs.hpp" #include "util/vector_view.hpp" -#include "storage/io_fwd.hpp" +#include "storage/tar_fwd.hpp" #include "storage/shared_memory_ownership.hpp" #include @@ -34,9 +34,9 @@ using MultiLevelPartitionView = detail::MultiLevelPartitionImpl -void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl &mlp); +void read(storage::tar::FileReader &reader, const std::string& name, detail::MultiLevelPartitionImpl &mlp); template -void write(storage::io::FileWriter &writer, const detail::MultiLevelPartitionImpl &mlp); +void write(storage::tar::FileWriter &writer, const std::string& name, const detail::MultiLevelPartitionImpl &mlp); } namespace detail @@ -136,9 +136,11 @@ template class MultiLevelPartitionImpl final return cell_to_children[offset + cell + 1]; } - friend void serialization::read(storage::io::FileReader &reader, + friend void serialization::read(storage::tar::FileReader &reader, + const std::string& name, MultiLevelPartitionImpl &mlp); - friend void serialization::write(storage::io::FileWriter &writer, + friend void serialization::write(storage::tar::FileWriter &writer, + const std::string& name, const MultiLevelPartitionImpl &mlp); private: diff --git a/include/partitioner/serialization.hpp b/include/partitioner/serialization.hpp index 4b33b8dbc..c92e1d0bd 100644 --- a/include/partitioner/serialization.hpp +++ b/include/partitioner/serialization.hpp @@ -8,9 +8,9 @@ #include "storage/block.hpp" #include "storage/io.hpp" -#include "storage/tar.hpp" #include "storage/serialization.hpp" #include "storage/shared_memory_ownership.hpp" +#include "storage/tar.hpp" namespace osrm { @@ -21,7 +21,7 @@ namespace serialization template inline void read(storage::tar::FileReader &reader, - const std::string& name, + const std::string &name, MultiLevelGraph &graph, std::uint32_t &connectivity_checksum) { @@ -33,7 +33,7 @@ inline void read(storage::tar::FileReader &reader, template inline void write(storage::tar::FileWriter &writer, - const std::string& name, + const std::string &name, const MultiLevelGraph &graph, const std::uint32_t connectivity_checksum) { @@ -45,39 +45,50 @@ inline void write(storage::tar::FileWriter &writer, } template -inline void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl &mlp) +inline void read(storage::tar::FileReader &reader, + const std::string &name, + detail::MultiLevelPartitionImpl &mlp) { - reader.ReadInto(*mlp.level_data); - storage::serialization::read(reader, mlp.partition); - storage::serialization::read(reader, mlp.cell_to_children); + *mlp.level_data = reader.ReadOne::LevelData>(name + "/level_data"); + storage::serialization::read(reader, name + "/partition", mlp.partition); + storage::serialization::read(reader, name + "/cell_to_children", mlp.cell_to_children); } template -inline void write(storage::io::FileWriter &writer, +inline void write(storage::tar::FileWriter &writer, + const std::string &name, const detail::MultiLevelPartitionImpl &mlp) { - writer.WriteOne(*mlp.level_data); - storage::serialization::write(writer, mlp.partition); - storage::serialization::write(writer, mlp.cell_to_children); + writer.WriteElementCount64(name + "/level_data", 1); + writer.WriteOne(name + "/level_data", *mlp.level_data); + storage::serialization::write(writer, name + "/partition", mlp.partition); + storage::serialization::write(writer, name + "/cell_to_children", mlp.cell_to_children); } template -inline void read(storage::io::FileReader &reader, detail::CellStorageImpl &storage) +inline void read(storage::tar::FileReader &reader, + const std::string &name, + detail::CellStorageImpl &storage) { - storage::serialization::read(reader, storage.source_boundary); - storage::serialization::read(reader, storage.destination_boundary); - storage::serialization::read(reader, storage.cells); - storage::serialization::read(reader, storage.level_to_cell_offset); + storage::serialization::read(reader, name + "/source_boundary", storage.source_boundary); + storage::serialization::read( + reader, name + "/destination_boundary", storage.destination_boundary); + storage::serialization::read(reader, name + "/cells", storage.cells); + storage::serialization::read( + reader, name + "/level_to_cell_offset", storage.level_to_cell_offset); } template -inline void write(storage::io::FileWriter &writer, +inline void write(storage::tar::FileWriter &writer, + const std::string &name, const detail::CellStorageImpl &storage) { - storage::serialization::write(writer, storage.source_boundary); - storage::serialization::write(writer, storage.destination_boundary); - storage::serialization::write(writer, storage.cells); - storage::serialization::write(writer, storage.level_to_cell_offset); + storage::serialization::write(writer, name + "/source_boundary", storage.source_boundary); + storage::serialization::write( + writer, name + "/destination_boundary", storage.destination_boundary); + storage::serialization::write(writer, name + "/cells", storage.cells); + storage::serialization::write( + writer, name + "/level_to_cell_offset", storage.level_to_cell_offset); } } } diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 343d0ba94..787b3a357 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -69,30 +69,28 @@ namespace storage { namespace { - template - void readBlocks(const boost::filesystem::path& path, OutIter out) +template void readBlocks(const boost::filesystem::path &path, OutIter out) +{ + tar::FileReader reader(path, tar::FileReader::VerifyFingerprint); + + std::vector entries; + reader.List(std::back_inserter(entries)); + + for (const auto &entry : entries) { - tar::FileReader reader(path, tar::FileReader::VerifyFingerprint); + std::string name; + std::uint64_t size; + std::tie(name, size) = entry; - std::vector entries; - reader.List(std::back_inserter(entries)); - - for (const auto& entry : entries) + const auto name_end = name.rfind(".meta"); + if (name_end == std::string::npos) { - std::string name; - std::uint64_t size; - std::tie(name, size) = entry; - - const auto name_end = name.rfind(".meta"); - if (name_end == std::string::npos) - { - auto number_of_elements = reader.ReadElementCount64(name); - *out++ = NamedBlock {name, Block{number_of_elements, size}}; - } + auto number_of_elements = reader.ReadElementCount64(name); + *out++ = NamedBlock{name, Block{number_of_elements, size}}; } - } } +} static constexpr std::size_t NUM_METRICS = 8; @@ -496,55 +494,6 @@ void Storage::PopulateLayout(DataLayout &layout) } { - // Loading MLD Data - if (boost::filesystem::exists(config.GetPath(".osrm.partition"))) - { - io::FileReader reader(config.GetPath(".osrm.partition"), - io::FileReader::VerifyFingerprint); - - reader.Skip(1); - layout.SetBlock(DataLayout::MLD_LEVEL_DATA, - make_block(1)); - const auto partition_entries_count = reader.ReadVectorSize(); - layout.SetBlock(DataLayout::MLD_PARTITION, - make_block(partition_entries_count)); - const auto children_entries_count = reader.ReadVectorSize(); - layout.SetBlock(DataLayout::MLD_CELL_TO_CHILDREN, - make_block(children_entries_count)); - } - else - { - layout.SetBlock(DataLayout::MLD_LEVEL_DATA, - make_block(0)); - layout.SetBlock(DataLayout::MLD_PARTITION, make_block(0)); - layout.SetBlock(DataLayout::MLD_CELL_TO_CHILDREN, make_block(0)); - } - - if (boost::filesystem::exists(config.GetPath(".osrm.cells"))) - { - io::FileReader reader(config.GetPath(".osrm.cells"), io::FileReader::VerifyFingerprint); - - const auto source_node_count = reader.ReadVectorSize(); - layout.SetBlock(DataLayout::MLD_CELL_SOURCE_BOUNDARY, - make_block(source_node_count)); - const auto destination_node_count = reader.ReadVectorSize(); - layout.SetBlock(DataLayout::MLD_CELL_DESTINATION_BOUNDARY, - make_block(destination_node_count)); - const auto cell_count = reader.ReadVectorSize(); - layout.SetBlock(DataLayout::MLD_CELLS, - make_block(cell_count)); - const auto level_offsets_count = reader.ReadVectorSize(); - layout.SetBlock(DataLayout::MLD_CELL_LEVEL_OFFSETS, - make_block(level_offsets_count)); - } - else - { - layout.SetBlock(DataLayout::MLD_CELL_SOURCE_BOUNDARY, make_block(0)); - layout.SetBlock(DataLayout::MLD_CELL_DESTINATION_BOUNDARY, make_block(0)); - layout.SetBlock(DataLayout::MLD_CELLS, make_block(0)); - layout.SetBlock(DataLayout::MLD_CELL_LEVEL_OFFSETS, make_block(0)); - } - if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics"))) { io::FileReader reader(config.GetPath(".osrm.cell_metrics"), @@ -603,20 +552,35 @@ void Storage::PopulateLayout(DataLayout &layout) {"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST}, {"/mld/multilevelgraph/node_to_edge_offset", DataLayout::MLD_GRAPH_NODE_TO_OFFSET}, {"/mld/multilevelgraph/connectivity_checksum", DataLayout::IGNORE_BLOCK}, + {"/mld/multilevelpartition/level_data", DataLayout::MLD_LEVEL_DATA}, + {"/mld/multilevelpartition/partition", DataLayout::MLD_PARTITION}, + {"/mld/multilevelpartition/cell_to_children", DataLayout::MLD_CELL_TO_CHILDREN}, + {"/mld/cellstorage/source_boundary", DataLayout::MLD_CELL_SOURCE_BOUNDARY}, + {"/mld/cellstorage/destination_boundary", DataLayout::MLD_CELL_DESTINATION_BOUNDARY}, + {"/mld/cellstorage/cells", DataLayout::MLD_CELLS}, + {"/mld/cellstorage/level_to_cell_offset", DataLayout::MLD_CELL_LEVEL_OFFSETS}, }; std::vector blocks; - if (boost::filesystem::exists(config.GetPath(".osrm.mldgr"))) + std::vector optional_tar_files = {config.GetPath(".osrm.mldgr"), + config.GetPath(".osrm.cells"), + config.GetPath(".osrm.partition")}; + + for (const auto &path : optional_tar_files) { - readBlocks(config.GetPath(".osrm.mldgr"), std::back_inserter(blocks)); + if (boost::filesystem::exists(path)) + { + readBlocks(path, std::back_inserter(blocks)); + } } - for (const auto& block : blocks) + for (const auto &block : blocks) { auto id_iter = name_to_block_id.find(std::get<0>(block)); if (id_iter == name_to_block_id.end()) { - throw util::exception("Could not map " + std::get<0>(block) + " to a region in memory."); + throw util::exception("Could not map " + std::get<0>(block) + + " to a region in memory."); } layout.SetBlock(id_iter->second, std::get<1>(block)); }