From a594008e57c729079627323645b3bf725f40ee4d Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 20 Mar 2018 01:05:11 +0000 Subject: [PATCH] Port .edges file to tar --- include/guidance/files.hpp | 12 +++---- include/guidance/serialization.hpp | 43 +++++++++++++++--------- include/guidance/turn_data_container.hpp | 14 +++++--- src/storage/storage.cpp | 24 ++++--------- 4 files changed, 50 insertions(+), 43 deletions(-) diff --git a/include/guidance/files.hpp b/include/guidance/files.hpp index 770104011..5fd77479d 100644 --- a/include/guidance/files.hpp +++ b/include/guidance/files.hpp @@ -27,10 +27,10 @@ inline void readTurnData(const boost::filesystem::path &path, std::is_same::value || std::is_same::value, ""); - const auto fingerprint = storage::io::FileReader::VerifyFingerprint; - storage::io::FileReader reader{path, fingerprint}; + const auto fingerprint = storage::tar::FileReader::VerifyFingerprint; + storage::tar::FileReader reader{path, fingerprint}; - serialization::read(reader, turn_data, connectivity_checksum); + serialization::read(reader, "/common/turn_data", turn_data, connectivity_checksum); } // writes .osrm.edges @@ -43,10 +43,10 @@ inline void writeTurnData(const boost::filesystem::path &path, std::is_same::value || std::is_same::value, ""); - const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; - storage::io::FileWriter writer{path, fingerprint}; + const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint; + storage::tar::FileWriter writer{path, fingerprint}; - serialization::write(writer, turn_data, connectivity_checksum); + serialization::write(writer, "/common/turn_data", turn_data, connectivity_checksum); } } } diff --git a/include/guidance/serialization.hpp b/include/guidance/serialization.hpp index 740c178d3..ea24f47df 100644 --- a/include/guidance/serialization.hpp +++ b/include/guidance/serialization.hpp @@ -3,7 +3,7 @@ #include "guidance/turn_data_container.hpp" -#include "storage/io.hpp" +#include "storage/tar.hpp" #include "storage/serialization.hpp" #include @@ -17,29 +17,42 @@ namespace serialization // read/write for turn data file template -inline void read(storage::io::FileReader &reader, +inline void read(storage::tar::FileReader &reader, + const std::string &name, guidance::detail::TurnDataContainerImpl &turn_data_container, std::uint32_t &connectivity_checksum) { - storage::serialization::read(reader, turn_data_container.turn_instructions); - storage::serialization::read(reader, turn_data_container.lane_data_ids); - storage::serialization::read(reader, turn_data_container.entry_class_ids); - storage::serialization::read(reader, turn_data_container.pre_turn_bearings); - storage::serialization::read(reader, turn_data_container.post_turn_bearings); - reader.ReadInto(connectivity_checksum); + storage::serialization::read( + reader, name + "/turn_instructions", turn_data_container.turn_instructions); + storage::serialization::read( + reader, name + "/lane_data_ids", turn_data_container.lane_data_ids); + storage::serialization::read( + reader, name + "/entry_class_ids", turn_data_container.entry_class_ids); + storage::serialization::read( + reader, name + "/pre_turn_bearings", turn_data_container.pre_turn_bearings); + storage::serialization::read( + reader, name + "/post_turn_bearings", turn_data_container.post_turn_bearings); + reader.ReadInto(name + "/connectivity_checksum", connectivity_checksum); } template -inline void write(storage::io::FileWriter &writer, +inline void write(storage::tar::FileWriter &writer, + const std::string &name, const guidance::detail::TurnDataContainerImpl &turn_data_container, const std::uint32_t connectivity_checksum) { - storage::serialization::write(writer, turn_data_container.turn_instructions); - storage::serialization::write(writer, turn_data_container.lane_data_ids); - storage::serialization::write(writer, turn_data_container.entry_class_ids); - storage::serialization::write(writer, turn_data_container.pre_turn_bearings); - storage::serialization::write(writer, turn_data_container.post_turn_bearings); - writer.WriteFrom(connectivity_checksum); + storage::serialization::write( + writer, name + "/turn_instructions", turn_data_container.turn_instructions); + storage::serialization::write( + writer, name + "/lane_data_ids", turn_data_container.lane_data_ids); + storage::serialization::write( + writer, name + "/entry_class_ids", turn_data_container.entry_class_ids); + storage::serialization::write( + writer, name + "/pre_turn_bearings", turn_data_container.pre_turn_bearings); + storage::serialization::write( + writer, name + "/post_turn_bearings", turn_data_container.post_turn_bearings); + writer.WriteElementCount64(name + "/connectivity_checksum", 1); + writer.WriteFrom(name + "/connectivity_checksum", connectivity_checksum); } } } diff --git a/include/guidance/turn_data_container.hpp b/include/guidance/turn_data_container.hpp index 7f9501d38..2f4b68643 100644 --- a/include/guidance/turn_data_container.hpp +++ b/include/guidance/turn_data_container.hpp @@ -5,8 +5,8 @@ #include "guidance/turn_bearing.hpp" #include "guidance/turn_instruction.hpp" -#include "storage/io_fwd.hpp" #include "storage/shared_memory_ownership.hpp" +#include "storage/tar_fwd.hpp" #include "util/vector_view.hpp" @@ -24,12 +24,14 @@ template class TurnDataContainerImpl; namespace serialization { template -void read(storage::io::FileReader &reader, +void read(storage::tar::FileReader &reader, + const std::string &name, detail::TurnDataContainerImpl &turn_data, std::uint32_t &connectivity_checksum); template -void write(storage::io::FileWriter &writer, +void write(storage::tar::FileWriter &writer, + const std::string &name, const detail::TurnDataContainerImpl &turn_data, const std::uint32_t connectivity_checksum); } @@ -97,10 +99,12 @@ template class TurnDataContainerImpl others.begin(), others.end(), [this](const TurnData &other) { push_back(other); }); } - friend void serialization::read(storage::io::FileReader &reader, + friend void serialization::read(storage::tar::FileReader &reader, + const std::string &name, TurnDataContainerImpl &turn_data_container, std::uint32_t &connectivity_checksum); - friend void serialization::write(storage::io::FileWriter &writer, + friend void serialization::write(storage::tar::FileWriter &writer, + const std::string &name, const TurnDataContainerImpl &turn_data_container, const std::uint32_t connectivity_checksum); diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index bc80e6d51..e025e2cf0 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -250,23 +250,6 @@ void Storage::PopulateLayout(DataLayout &layout) layout.SetBlock(DataLayout::NAME_CHAR_DATA, make_block(name_file.GetSize())); } - // Loading information for original edges - { - io::FileReader edges_file(config.GetPath(".osrm.edges"), io::FileReader::VerifyFingerprint); - const auto number_of_original_edges = edges_file.ReadElementCount64(); - - // note: settings this all to the same size is correct, we extract them from the same struct - layout.SetBlock(DataLayout::PRE_TURN_BEARING, - make_block(number_of_original_edges)); - layout.SetBlock(DataLayout::POST_TURN_BEARING, - make_block(number_of_original_edges)); - layout.SetBlock(DataLayout::TURN_INSTRUCTION, - make_block(number_of_original_edges)); - layout.SetBlock(DataLayout::LANE_DATA_ID, make_block(number_of_original_edges)); - layout.SetBlock(DataLayout::ENTRY_CLASSID, - make_block(number_of_original_edges)); - } - // load rsearch tree size { io::FileReader tree_node_file(config.GetPath(".osrm.ramIndex"), @@ -356,6 +339,12 @@ void Storage::PopulateLayout(DataLayout &layout) {"/common/maneuver_overrides/node_sequences", DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES}, {"/common/turn_penalty/weight", DataLayout::TURN_WEIGHT_PENALTIES}, {"/common/turn_penalty/duration", DataLayout::TURN_DURATION_PENALTIES}, + {"/common/turn_data/pre_turn_bearings", DataLayout::PRE_TURN_BEARING}, + {"/common/turn_data/post_turn_bearings", DataLayout::POST_TURN_BEARING}, + {"/common/turn_data/turn_instructions", DataLayout::TURN_INSTRUCTION}, + {"/common/turn_data/lane_data_ids", DataLayout::LANE_DATA_ID}, + {"/common/turn_data/entry_class_ids", DataLayout::ENTRY_CLASSID}, + {"/common/turn_data/connectivity_checksum", DataLayout::IGNORE_BLOCK}, }; std::vector blocks; @@ -378,6 +367,7 @@ void Storage::PopulateLayout(DataLayout &layout) {REQUIRED, config.GetPath(".osrm.maneuver_overrides")}, {REQUIRED, config.GetPath(".osrm.turn_weight_penalties")}, {REQUIRED, config.GetPath(".osrm.turn_duration_penalties")}, + {REQUIRED, config.GetPath(".osrm.edges")}, }; for (const auto &file : tar_files)