diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 74095ef6d..273b9c73b 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -116,11 +116,11 @@ inline void readNodes(const boost::filesystem::path &path, static_assert(std::is_same::value, ""); static_assert(std::is_same::value, ""); - const auto fingerprint = storage::io::FileReader::VerifyFingerprint; - storage::io::FileReader reader{path, fingerprint}; + const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; + storage::tar::FileReader reader{path, fingerprint}; - storage::serialization::read(reader, coordinates); - util::serialization::read(reader, osm_node_ids); + storage::serialization::read(reader, "/common/coordinates", coordinates); + util::serialization::read(reader, "/common/osm_node_ids", osm_node_ids); } // writes .osrm.nodes @@ -132,11 +132,11 @@ inline void writeNodes(const boost::filesystem::path &path, static_assert(std::is_same::value, ""); static_assert(std::is_same::value, ""); - const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; - storage::io::FileWriter writer{path, fingerprint}; + const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; + storage::tar::FileWriter writer{path, fingerprint}; - storage::serialization::write(writer, coordinates); - util::serialization::write(writer, osm_node_ids); + storage::serialization::write(writer, "/common/coordinates", coordinates); + util::serialization::write(writer, "/common/osm_node_ids", osm_node_ids); } // reads .osrm.cnbg_to_ebg diff --git a/include/util/packed_vector.hpp b/include/util/packed_vector.hpp index e18a23821..20f7c7ba4 100644 --- a/include/util/packed_vector.hpp +++ b/include/util/packed_vector.hpp @@ -7,6 +7,7 @@ #include "storage/io_fwd.hpp" #include "storage/shared_memory_ownership.hpp" +#include "storage/tar_fwd.hpp" #include #include @@ -34,6 +35,16 @@ inline void read(storage::io::FileReader &reader, detail::PackedVector inline void write(storage::io::FileWriter &writer, const detail::PackedVector &vec); + +template +inline void read(storage::tar::FileReader &reader, + const std::string &name, + detail::PackedVector &vec); + +template +inline void write(storage::tar::FileWriter &writer, + const std::string &name, + const detail::PackedVector &vec); } namespace detail @@ -451,6 +462,15 @@ template class Pack friend void serialization::write(storage::io::FileWriter &writer, const PackedVector &vec); + + friend void serialization::read(storage::tar::FileReader &reader, + const std::string &name, + PackedVector &vec); + + friend void serialization::write(storage::tar::FileWriter &writer, + const std::string &name, + const PackedVector &vec); + inline void swap(PackedVector &other) noexcept { std::swap(vec, other.vec); diff --git a/include/util/serialization.hpp b/include/util/serialization.hpp index 68cc96f17..b92ebe44d 100644 --- a/include/util/serialization.hpp +++ b/include/util/serialization.hpp @@ -32,7 +32,9 @@ void read(storage::io::FileReader &reader, util::RangeTable -void write(storage::tar::FileWriter &writer, const std::string& name, const util::RangeTable &table) +void write(storage::tar::FileWriter &writer, + const std::string &name, + const util::RangeTable &table) { writer.WriteOne(name + "/sum_lengths.meta", table.sum_lengths); storage::serialization::write(writer, name + "/block_offsets", table.block_offsets); @@ -40,7 +42,9 @@ void write(storage::tar::FileWriter &writer, const std::string& name, const util } template -void read(storage::tar::FileReader &reader, const std::string& name, util::RangeTable &table) +void read(storage::tar::FileReader &reader, + const std::string &name, + util::RangeTable &table) { table.sum_lengths = reader.ReadOne(name + "/sum_lengths.meta"); storage::serialization::read(reader, name + "/block_offsets", table.block_offsets); @@ -62,6 +66,24 @@ inline void write(storage::io::FileWriter &writer, storage::serialization::write(writer, vec.vec); } +template +inline void read(storage::tar::FileReader &reader, + const std::string &name, + detail::PackedVector &vec) +{ + vec.num_elements = reader.ReadOne(name + "/number_of_elements.meta"); + storage::serialization::read(reader, name + "/packed", vec.vec); +} + +template +inline void write(storage::tar::FileWriter &writer, + const std::string &name, + const detail::PackedVector &vec) +{ + writer.WriteOne(name + "/number_of_elements.meta", vec.num_elements); + storage::serialization::write(writer, name + "/packed", vec.vec); +} + template inline void read(storage::io::FileReader &reader, StaticGraph &graph) { diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 6e74bda27..3e16ab351 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -328,23 +328,6 @@ void Storage::PopulateLayout(DataLayout &layout) make_block(number_of_penalties)); } - // load coordinate size - { - io::FileReader node_file(config.GetPath(".osrm.nbg_nodes"), - io::FileReader::VerifyFingerprint); - const auto coordinate_list_size = node_file.ReadElementCount64(); - layout.SetBlock(DataLayout::COORDINATE_LIST, - make_block(coordinate_list_size)); - node_file.Skip(coordinate_list_size); - // skip number of elements - node_file.Skip(1); - const auto num_id_blocks = node_file.ReadElementCount64(); - // we'll read a list of OSM node IDs from the same data, so set the block size for the same - // number of items: - layout.SetBlock(DataLayout::OSM_NODE_ID_LIST, - make_block(num_id_blocks)); - } - // load geometries sizes { io::FileReader reader(config.GetPath(".osrm.geometry"), io::FileReader::VerifyFingerprint); @@ -462,6 +445,8 @@ void Storage::PopulateLayout(DataLayout &layout) {"/common/intersection_bearings/class_id_to_ranges/diff_blocks", DataLayout::BEARING_BLOCKS}, {"/common/entry_classes", DataLayout::ENTRY_CLASS}, {"/common/properties", DataLayout::PROPERTIES}, + {"/common/coordinates", DataLayout::COORDINATE_LIST}, + {"/common/osm_node_ids/packed", DataLayout::OSM_NODE_ID_LIST}, }; std::vector blocks; @@ -474,7 +459,8 @@ void Storage::PopulateLayout(DataLayout &layout) {OPTIONAL, config.GetPath(".osrm.cell_metrics")}, {OPTIONAL, config.GetPath(".osrm.hsgr")}, {REQUIRED, config.GetPath(".osrm.icd")}, - {REQUIRED, config.GetPath(".osrm.properties")} + {REQUIRED, config.GetPath(".osrm.properties")}, + {REQUIRED, config.GetPath(".osrm.nbg_nodes")} }; for (const auto &file : tar_files)