From 5d1b4ce71d48d876c13424bb1b1b4cb602962aac Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 16 Mar 2018 16:39:18 +0000 Subject: [PATCH] Port .osrm.icd file over to tar --- CMakeLists.txt | 2 +- include/extractor/files.hpp | 12 +++--- .../intersection_bearings_container.hpp | 13 +++++-- include/extractor/serialization.hpp | 18 +++++---- include/util/range_table.hpp | 16 +++++--- include/util/serialization.hpp | 16 ++++++++ src/storage/storage.cpp | 32 ++++------------ unit_tests/util/serialization.cpp | 38 +++++++++++++++++++ 8 files changed, 98 insertions(+), 49 deletions(-) create mode 100644 unit_tests/util/serialization.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8838aa2f5..2a85b681e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,7 +169,7 @@ add_executable(osrm-routed src/tools/routed.cpp $ $ $) add_library(osrm src/osrm/osrm.cpp $ $ $ $ ) add_library(osrm_contract src/osrm/contractor.cpp $ $) -add_library(osrm_extract src/osrm/extractor.cpp $ $) +add_library(osrm_extract src/osrm/extractor.cpp $ $ $) add_library(osrm_guidance $ $) add_library(osrm_partition src/osrm/partitioner.cpp $ $ $) add_library(osrm_customize src/osrm/customizer.cpp $ $ $) diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index bc5e32c51..eca87eaf1 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -33,10 +33,10 @@ inline void writeIntersections(const boost::filesystem::path &path, std::is_same::value, ""); - storage::io::FileWriter writer(path, storage::io::FileWriter::GenerateFingerprint); + storage::tar::FileWriter writer(path, storage::tar::FileWriter::GenerateFingerprint); - serialization::write(writer, intersection_bearings); - storage::serialization::write(writer, entry_classes); + serialization::write(writer, "/common/intersection_bearings", intersection_bearings); + storage::serialization::write(writer, "/common/entry_classes", entry_classes); } // read the .osrm.icd file @@ -49,10 +49,10 @@ inline void readIntersections(const boost::filesystem::path &path, std::is_same::value, ""); - storage::io::FileReader reader(path, storage::io::FileReader::VerifyFingerprint); + storage::tar::FileReader reader(path, storage::tar::FileReader::VerifyFingerprint); - serialization::read(reader, intersection_bearings); - storage::serialization::read(reader, entry_classes); + serialization::read(reader, "/common/intersection_bearings", intersection_bearings); + storage::serialization::read(reader, "/common/entry_classes", entry_classes); } // reads .osrm.properties diff --git a/include/extractor/intersection_bearings_container.hpp b/include/extractor/intersection_bearings_container.hpp index 8e25a0711..cdf884141 100644 --- a/include/extractor/intersection_bearings_container.hpp +++ b/include/extractor/intersection_bearings_container.hpp @@ -2,6 +2,7 @@ #define OSRM_EXTRACTOR_BEARING_CONTAINER_HPP #include "storage/shared_memory_ownership.hpp" +#include "storage/tar_fwd.hpp" #include "util/guidance/bearing_class.hpp" #include "util/range_table.hpp" @@ -21,11 +22,13 @@ template class IntersectionBearingsContainer; namespace serialization { template -void read(storage::io::FileReader &reader, +void read(storage::tar::FileReader &reader, + const std::string& name, detail::IntersectionBearingsContainer &turn_data); template -void write(storage::io::FileWriter &writer, +void write(storage::tar::FileWriter &writer, + const std::string& name, const detail::IntersectionBearingsContainer &turn_data); } @@ -83,10 +86,12 @@ template class IntersectionBearingsContainer return result; } - friend void serialization::read(storage::io::FileReader &reader, + friend void serialization::read(storage::tar::FileReader &reader, + const std::string &name, IntersectionBearingsContainer &turn_data_container); friend void - serialization::write(storage::io::FileWriter &writer, + serialization::write(storage::tar::FileWriter &writer, + const std::string &name, const IntersectionBearingsContainer &turn_data_container); private: diff --git a/include/extractor/serialization.hpp b/include/extractor/serialization.hpp index d5ed0cbc0..04dd1f531 100644 --- a/include/extractor/serialization.hpp +++ b/include/extractor/serialization.hpp @@ -25,21 +25,23 @@ namespace serialization // read/write for bearing data template -inline void read(storage::io::FileReader &reader, +inline void read(storage::tar::FileReader &reader, + const std::string& name, detail::IntersectionBearingsContainer &intersection_bearings) { - storage::serialization::read(reader, intersection_bearings.values); - storage::serialization::read(reader, intersection_bearings.node_to_class_id); - util::serialization::read(reader, intersection_bearings.class_id_to_ranges_table); + storage::serialization::read(reader, name + "/bearing_values", intersection_bearings.values); + storage::serialization::read(reader, name + "/node_to_class_id", intersection_bearings.node_to_class_id); + util::serialization::read(reader, name + "/class_id_to_ranges", intersection_bearings.class_id_to_ranges_table); } template -inline void write(storage::io::FileWriter &writer, +inline void write(storage::tar::FileWriter &writer, + const std::string& name, const detail::IntersectionBearingsContainer &intersection_bearings) { - storage::serialization::write(writer, intersection_bearings.values); - storage::serialization::write(writer, intersection_bearings.node_to_class_id); - util::serialization::write(writer, intersection_bearings.class_id_to_ranges_table); + storage::serialization::write(writer, name + "/bearing_values", intersection_bearings.values); + storage::serialization::write(writer, name + "/node_to_class_id", intersection_bearings.node_to_class_id); + util::serialization::write(writer, name + "/class_id_to_ranges", intersection_bearings.class_id_to_ranges_table); } // read/write for properties file diff --git a/include/util/range_table.hpp b/include/util/range_table.hpp index 33448711a..cceee6713 100644 --- a/include/util/range_table.hpp +++ b/include/util/range_table.hpp @@ -1,8 +1,8 @@ #ifndef RANGE_TABLE_HPP #define RANGE_TABLE_HPP -#include "storage/io.hpp" #include "storage/shared_memory_ownership.hpp" +#include "storage/tar_fwd.hpp" #include "util/integer_range.hpp" #include "util/vector_view.hpp" @@ -25,10 +25,14 @@ class RangeTable; namespace serialization { template -void write(storage::io::FileWriter &writer, const util::RangeTable &table); +void write(storage::tar::FileWriter &writer, + const std::string &name, + const util::RangeTable &table); template -void read(storage::io::FileReader &reader, util::RangeTable &table); +void read(storage::tar::FileReader &reader, + const std::string &name, + util::RangeTable &table); } /** @@ -177,9 +181,11 @@ template class RangeTable return irange(begin_idx, end_idx); } - friend void serialization::write(storage::io::FileWriter &writer, + friend void serialization::write(storage::tar::FileWriter &writer, + const std::string &name, const RangeTable &table); - friend void serialization::read(storage::io::FileReader &reader, + friend void serialization::read(storage::tar::FileReader &reader, + const std::string &name, RangeTable &table); private: diff --git a/include/util/serialization.hpp b/include/util/serialization.hpp index ccb9e7f69..68cc96f17 100644 --- a/include/util/serialization.hpp +++ b/include/util/serialization.hpp @@ -31,6 +31,22 @@ void read(storage::io::FileReader &reader, util::RangeTable +void write(storage::tar::FileWriter &writer, const std::string& name, const util::RangeTable &table) +{ + writer.WriteOne(name + "/sum_lengths.meta", table.sum_lengths); + storage::serialization::write(writer, name + "/block_offsets", table.block_offsets); + storage::serialization::write(writer, name + "/diff_blocks", table.diff_blocks); +} + +template +void read(storage::tar::FileReader &reader, const std::string& name, util::RangeTable &table) +{ + table.sum_lengths = reader.ReadOne(name + "/sum_lengths.meta"); + storage::serialization::read(reader, name + "/block_offsets", table.block_offsets); + storage::serialization::read(reader, name + "/diff_blocks", table.diff_blocks); +} + template inline void read(storage::io::FileReader &reader, detail::PackedVector &vec) { diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 109388f7d..1a590eeec 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -399,30 +399,6 @@ void Storage::PopulateLayout(DataLayout &layout) } { - io::FileReader reader(config.GetPath(".osrm.icd"), io::FileReader::VerifyFingerprint); - - auto num_discreate_bearings = reader.ReadVectorSize(); - layout.SetBlock(DataLayout::BEARING_VALUES, - make_block(num_discreate_bearings)); - - auto num_bearing_classes = reader.ReadVectorSize(); - layout.SetBlock(DataLayout::BEARING_CLASSID, - make_block(num_bearing_classes)); - - reader.Skip(1); // sum_lengths - const auto bearing_blocks = reader.ReadVectorSize(); - const auto bearing_offsets = - reader - .ReadVectorSize::BlockT>(); - - layout.SetBlock(DataLayout::BEARING_OFFSETS, make_block(bearing_blocks)); - layout.SetBlock(DataLayout::BEARING_BLOCKS, - make_block::BlockT>( - bearing_offsets)); - - auto num_entry_classes = reader.ReadVectorSize(); - layout.SetBlock(DataLayout::ENTRY_CLASS, - make_block(num_entry_classes)); } { @@ -487,6 +463,11 @@ void Storage::PopulateLayout(DataLayout &layout) {"/ch/edge_filter/5", DataLayout::CH_EDGE_FILTER_5}, {"/ch/edge_filter/6", DataLayout::CH_EDGE_FILTER_6}, {"/ch/edge_filter/7", DataLayout::CH_EDGE_FILTER_7}, + {"/common/intersection_bearings/bearing_values", DataLayout::BEARING_VALUES}, + {"/common/intersection_bearings/node_to_class_id", DataLayout::BEARING_CLASSID}, + {"/common/intersection_bearings/class_id_to_ranges/block_offsets", DataLayout::BEARING_OFFSETS}, + {"/common/intersection_bearings/class_id_to_ranges/diff_blocks", DataLayout::BEARING_BLOCKS}, + {"/common/entry_classes", DataLayout::ENTRY_CLASS}, }; std::vector blocks; @@ -497,7 +478,8 @@ void Storage::PopulateLayout(DataLayout &layout) {OPTIONAL, config.GetPath(".osrm.cells")}, {OPTIONAL, config.GetPath(".osrm.partition")}, {OPTIONAL, config.GetPath(".osrm.cell_metrics")}, - {OPTIONAL, config.GetPath(".osrm.hsgr")} + {OPTIONAL, config.GetPath(".osrm.hsgr")}, + {REQUIRED, config.GetPath(".osrm.icd")} }; for (const auto &file : tar_files) diff --git a/unit_tests/util/serialization.cpp b/unit_tests/util/serialization.cpp new file mode 100644 index 000000000..64c92ab6a --- /dev/null +++ b/unit_tests/util/serialization.cpp @@ -0,0 +1,38 @@ +#include "util/serialization.hpp" + +#include "../common/range_tools.hpp" +#include "../common/temporary_file.hpp" + +#include +#include + +BOOST_AUTO_TEST_SUITE(serialization) + +using namespace osrm; +using namespace osrm::util; + +BOOST_AUTO_TEST_CASE(tar_serialize_range_table) +{ + TemporaryFile tmp; + { + constexpr unsigned BLOCK_SIZE = 16; + using TestRangeTable = RangeTable; + + std::vector data = { + }; + + for (const auto &v : data) + { + { + storage::tar::FileWriter writer(tmp.path, storage::tar::FileWriter::GenerateFingerprint); + util::serialization::write(writer, "my_range_table", v); + } + + TestRangeTable result; + storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint); + util::serialization::read(reader, "my_range_table", result); + } + } +} + +BOOST_AUTO_TEST_SUITE_END()