From a542da367879f2622bb6db2d733863672101d8d1 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 22 Mar 2018 17:34:26 +0000 Subject: [PATCH] Remove old io::FileWrite serialization code --- include/extractor/serialization.hpp | 137 --------------------------- include/storage/serialization.hpp | 135 ++------------------------ include/util/deallocating_vector.hpp | 24 +---- include/util/dynamic_graph.hpp | 13 --- include/util/packed_vector.hpp | 14 --- include/util/serialization.hpp | 71 -------------- include/util/static_graph.hpp | 13 --- unit_tests/storage/serialization.cpp | 20 ---- unit_tests/util/io.cpp | 18 ++-- 9 files changed, 20 insertions(+), 425 deletions(-) diff --git a/include/extractor/serialization.hpp b/include/extractor/serialization.hpp index e388dc8cb..bae2f44a5 100644 --- a/include/extractor/serialization.hpp +++ b/include/extractor/serialization.hpp @@ -132,143 +132,6 @@ inline void write(storage::tar::FileWriter &writer, writer, name + "/annotations", node_data_container.annotation_data); } -inline void read(storage::io::FileReader &reader, NodeRestriction &restriction) -{ - reader.ReadInto(restriction.from); - reader.ReadInto(restriction.via); - reader.ReadInto(restriction.to); -} - -inline void write(storage::io::FileWriter &writer, const NodeRestriction &restriction) -{ - writer.WriteFrom(restriction.from); - writer.WriteFrom(restriction.via); - writer.WriteFrom(restriction.to); -} - -inline void read(storage::io::FileReader &reader, WayRestriction &restriction) -{ - read(reader, restriction.in_restriction); - read(reader, restriction.out_restriction); -} - -inline void write(storage::io::FileWriter &writer, const WayRestriction &restriction) -{ - write(writer, restriction.in_restriction); - write(writer, restriction.out_restriction); -} - -inline void read(storage::io::FileReader &reader, TurnRestriction &restriction) -{ - reader.ReadInto(restriction.is_only); - std::uint32_t restriction_type; - reader.ReadInto(restriction_type); - if (restriction_type == RestrictionType::WAY_RESTRICTION) - { - WayRestriction way_restriction; - read(reader, way_restriction); - restriction.node_or_way = std::move(way_restriction); - } - else - { - BOOST_ASSERT(restriction_type == RestrictionType::NODE_RESTRICTION); - NodeRestriction node_restriction; - read(reader, node_restriction); - restriction.node_or_way = std::move(node_restriction); - } -} - -inline void write(storage::io::FileWriter &writer, const TurnRestriction &restriction) -{ - writer.WriteFrom(restriction.is_only); - const std::uint32_t restriction_type = restriction.Type(); - writer.WriteFrom(restriction_type); - if (restriction.Type() == RestrictionType::WAY_RESTRICTION) - { - write(writer, mapbox::util::get(restriction.node_or_way)); - } - else - { - BOOST_ASSERT(restriction.Type() == RestrictionType::NODE_RESTRICTION); - write(writer, mapbox::util::get(restriction.node_or_way)); - } -} - -inline void write(storage::io::FileWriter &writer, const ConditionalTurnRestriction &restriction) -{ - write(writer, static_cast(restriction)); - writer.WriteElementCount64(restriction.condition.size()); - for (const auto &c : restriction.condition) - { - writer.WriteFrom(c.modifier); - storage::serialization::write(writer, c.times); - storage::serialization::write(writer, c.weekdays); - storage::serialization::write(writer, c.monthdays); - } -} - -inline void read(storage::io::FileReader &reader, ConditionalTurnRestriction &restriction) -{ - read(reader, static_cast(restriction)); - const auto num_conditions = reader.ReadElementCount64(); - restriction.condition.resize(num_conditions); - for (uint64_t i = 0; i < num_conditions; i++) - { - reader.ReadInto(restriction.condition[i].modifier); - storage::serialization::read(reader, restriction.condition[i].times); - storage::serialization::read(reader, restriction.condition[i].weekdays); - storage::serialization::read(reader, restriction.condition[i].monthdays); - } -} - -// read/write for conditional turn restrictions file -inline void read(storage::io::FileReader &reader, std::vector &restrictions) -{ - auto num_indices = reader.ReadElementCount64(); - restrictions.reserve(num_indices); - TurnRestriction restriction; - while (num_indices-- > 0) - { - read(reader, restriction); - restrictions.push_back(std::move(restriction)); - } -} - -inline void write(storage::io::FileWriter &writer, const std::vector &restrictions) -{ - const auto num_indices = restrictions.size(); - writer.WriteElementCount64(num_indices); - const auto write_restriction = [&writer](const auto &restriction) { - write(writer, restriction); - }; - std::for_each(restrictions.begin(), restrictions.end(), write_restriction); -} - -// read/write for conditional turn restrictions file -inline void read(storage::io::FileReader &reader, - std::vector &restrictions) -{ - auto num_indices = reader.ReadElementCount64(); - restrictions.reserve(num_indices); - ConditionalTurnRestriction restriction; - while (num_indices-- > 0) - { - read(reader, restriction); - restrictions.push_back(std::move(restriction)); - } -} - -inline void write(storage::io::FileWriter &writer, - const std::vector &restrictions) -{ - const auto num_indices = restrictions.size(); - writer.WriteElementCount64(num_indices); - const auto write_restriction = [&writer](const auto &restriction) { - write(writer, restriction); - }; - std::for_each(restrictions.begin(), restrictions.end(), write_restriction); -} - inline void read(storage::io::BufferReader &reader, ConditionalTurnPenalty &turn_penalty) { reader.ReadInto(turn_penalty.turn_offset); diff --git a/include/storage/serialization.hpp b/include/storage/serialization.hpp index 0b9f68a08..3b586a48a 100644 --- a/include/storage/serialization.hpp +++ b/include/storage/serialization.hpp @@ -32,41 +32,6 @@ namespace serialization * All vector types with this guarantee should be placed in this file. */ -template -inline void read(storage::io::FileReader &reader, util::DeallocatingVector &vec) -{ - vec.current_size = reader.ReadElementCount64(vec.current_size); - std::size_t num_blocks = - std::ceil(vec.current_size / util::DeallocatingVector::ELEMENTS_PER_BLOCK); - vec.bucket_list.resize(num_blocks); - // Read all but the last block which can be partital - for (auto bucket_index : util::irange(0, num_blocks - 1)) - { - vec.bucket_list[bucket_index] = new T[util::DeallocatingVector::ELEMENTS_PER_BLOCK]; - reader.ReadInto(vec.bucket_list[bucket_index], - util::DeallocatingVector::ELEMENTS_PER_BLOCK); - } - std::size_t last_block_size = - vec.current_size % util::DeallocatingVector::ELEMENTS_PER_BLOCK; - vec.bucket_list.back() = new T[util::DeallocatingVector::ELEMENTS_PER_BLOCK]; - reader.ReadInto(vec.bucket_list.back(), last_block_size); -} - -template -inline void write(storage::io::FileWriter &writer, const util::DeallocatingVector &vec) -{ - writer.WriteElementCount64(vec.current_size); - // Write all but the last block which can be partially filled - for (auto bucket_index : util::irange(0, vec.bucket_list.size() - 1)) - { - writer.WriteFrom(vec.bucket_list[bucket_index], - util::DeallocatingVector::ELEMENTS_PER_BLOCK); - } - std::size_t last_block_size = - vec.current_size % util::DeallocatingVector::ELEMENTS_PER_BLOCK; - writer.WriteFrom(vec.bucket_list.back(), last_block_size); -} - template inline void read(storage::tar::FileReader &reader, const std::string& name, util::DeallocatingVector &vec) { @@ -82,27 +47,18 @@ inline void write(storage::tar::FileWriter &writer, const std::string& name, con } #if USE_STXXL_LIBRARY -template inline void read(storage::io::FileReader &reader, stxxl::vector &vec) +template inline void read(storage::tar::FileReader &reader, const std::string& name, stxxl::vector &vec) { - auto size = reader.ReadElementCount64(); + auto size = reader.ReadElementCount64(name); vec.reserve(size); - T tmp; - for (auto idx : util::irange(0, size)) - { - (void)idx; - reader.ReadInto(tmp); - vec.push_back(tmp); - } + reader.ReadStreaming(name, std::back_inserter(vec), size); } template -inline void write(storage::io::FileWriter &writer, const stxxl::vector &vec) +inline void write(storage::tar::FileWriter &writer, const std::string& name, const stxxl::vector &vec) { - writer.WriteFrom(vec.size()); - for (auto idx : util::irange(0, vec.size())) - { - writer.WriteFrom(vec[idx]); - } + writer.WriteElementCount64(name, vec.size()); + writer.WriteStreaming(name, vec.begin(), vec.size()); } #endif @@ -168,34 +124,6 @@ void write(tar::FileWriter &writer, const std::string &name, const util::vector_ writer.WriteFrom(name, data.data(), count); } -template void read(io::FileReader &reader, std::vector &data) -{ - const auto count = reader.ReadElementCount64(); - data.resize(count); - reader.ReadInto(data.data(), count); -} - -template void write(io::FileWriter &writer, const std::vector &data) -{ - const auto count = data.size(); - writer.WriteElementCount64(count); - writer.WriteFrom(data.data(), count); -} - -template void read(io::FileReader &reader, util::vector_view &data) -{ - const auto count = reader.ReadElementCount64(); - data.resize(count); - reader.ReadInto(data.data(), count); -} - -template void write(io::FileWriter &writer, const util::vector_view &data) -{ - const auto count = data.size(); - writer.WriteElementCount64(count); - writer.WriteFrom(data.data(), count); -} - namespace detail { template @@ -217,37 +145,6 @@ inline void unpackBits(T &data, std::size_t index, std::size_t count, unsigned c data[index] = value & mask; } -template void readBoolVector(io::FileReader &reader, VectorT &data) -{ - const auto count = reader.ReadElementCount64(); - data.resize(count); - std::uint64_t index = 0; - unsigned char block_data; - for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT) - { - reader.ReadInto(block_data); - unpackBits(data, index, CHAR_BIT, block_data); - } - if (count > index) - { - reader.ReadInto(block_data); - unpackBits(data, index, count - index, block_data); - } -} - -template void writeBoolVector(io::FileWriter &writer, const VectorT &data) -{ - const auto count = data.size(); - writer.WriteElementCount64(count); - std::uint64_t index = 0; - for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT) - { - writer.WriteFrom(packBits(data, index, CHAR_BIT)); - } - if (count > index) - writer.WriteFrom(packBits(data, index, count - index)); -} - template void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data) { @@ -286,26 +183,6 @@ void writeBoolVector(tar::FileWriter &writer, const std::string &name, const Vec } } -template <> inline void read(io::FileReader &reader, util::vector_view &data) -{ - detail::readBoolVector(reader, data); -} - -template <> inline void write(io::FileWriter &writer, const util::vector_view &data) -{ - detail::writeBoolVector(writer, data); -} - -template <> inline void read(io::FileReader &reader, std::vector &data) -{ - detail::readBoolVector(reader, data); -} - -template <> inline void write(io::FileWriter &writer, const std::vector &data) -{ - detail::writeBoolVector(writer, data); -} - template <> inline void read(tar::FileReader &reader, const std::string &name, util::vector_view &data) diff --git a/include/util/deallocating_vector.hpp b/include/util/deallocating_vector.hpp index 7ffc0519e..1a160271b 100644 --- a/include/util/deallocating_vector.hpp +++ b/include/util/deallocating_vector.hpp @@ -13,23 +13,6 @@ namespace osrm { -namespace util -{ -template class DeallocatingVector; -} - -namespace storage -{ -namespace serialization -{ -template -inline void read(storage::io::FileReader &reader, util::DeallocatingVector &vec); - -template -inline void write(storage::io::FileWriter &writer, const util::DeallocatingVector &vec); -} -} - namespace util { template struct ConstDeallocatingVectorIteratorState @@ -183,6 +166,8 @@ class DeallocatingVectorIterator } }; +template class DeallocatingVector; + template void swap(DeallocatingVector &lhs, DeallocatingVector &rhs); template class DeallocatingVector @@ -359,11 +344,6 @@ template class DeallocatingVector ++position; } } - - friend void storage::serialization::read(storage::io::FileReader &reader, - DeallocatingVector &vec); - friend void storage::serialization::write(storage::io::FileWriter &writer, - const DeallocatingVector &vec); }; template void swap(DeallocatingVector &lhs, DeallocatingVector &rhs) diff --git a/include/util/dynamic_graph.hpp b/include/util/dynamic_graph.hpp index f2a3d8b3b..95eac6d07 100644 --- a/include/util/dynamic_graph.hpp +++ b/include/util/dynamic_graph.hpp @@ -6,8 +6,6 @@ #include "util/permutation.hpp" #include "util/typedefs.hpp" -#include "storage/io_fwd.hpp" - #include #include @@ -22,17 +20,6 @@ namespace osrm { namespace util { -template class DynamicGraph; - -namespace serialization -{ -template -void read(storage::io::FileReader &reader, DynamicGraph &graph); - -template -void write(storage::io::FileWriter &writer, const DynamicGraph &graph); -} - namespace detail { // These types need to live outside of DynamicGraph diff --git a/include/util/packed_vector.hpp b/include/util/packed_vector.hpp index 20f7c7ba4..cec8a1235 100644 --- a/include/util/packed_vector.hpp +++ b/include/util/packed_vector.hpp @@ -5,7 +5,6 @@ #include "util/typedefs.hpp" #include "util/vector_view.hpp" -#include "storage/io_fwd.hpp" #include "storage/shared_memory_ownership.hpp" #include "storage/tar_fwd.hpp" @@ -29,13 +28,6 @@ template class Pack namespace serialization { -template -inline void read(storage::io::FileReader &reader, detail::PackedVector &vec); - -template -inline void write(storage::io::FileWriter &writer, - const detail::PackedVector &vec); - template inline void read(storage::tar::FileReader &reader, const std::string &name, @@ -457,12 +449,6 @@ template class Pack vec.reserve(num_blocks * BLOCK_WORDS + 1); } - friend void serialization::read(storage::io::FileReader &reader, - PackedVector &vec); - - friend void serialization::write(storage::io::FileWriter &writer, - const PackedVector &vec); - friend void serialization::read(storage::tar::FileReader &reader, const std::string &name, PackedVector &vec); diff --git a/include/util/serialization.hpp b/include/util/serialization.hpp index 314497812..0eea6f1f4 100644 --- a/include/util/serialization.hpp +++ b/include/util/serialization.hpp @@ -17,21 +17,6 @@ namespace util { namespace serialization { -template -void write(storage::io::FileWriter &writer, const util::RangeTable &table) -{ - writer.WriteFrom(table.sum_lengths); - storage::serialization::write(writer, table.block_offsets); - storage::serialization::write(writer, table.diff_blocks); -} - -template -void read(storage::io::FileReader &reader, util::RangeTable &table) -{ - reader.ReadInto(table.sum_lengths); - storage::serialization::read(reader, table.block_offsets); - storage::serialization::read(reader, table.diff_blocks); -} template void write(storage::tar::FileWriter &writer, @@ -53,21 +38,6 @@ void read(storage::tar::FileReader &reader, storage::serialization::read(reader, name + "/diff_blocks", table.diff_blocks); } -template -inline void read(storage::io::FileReader &reader, detail::PackedVector &vec) -{ - reader.ReadInto(vec.num_elements); - storage::serialization::read(reader, vec.vec); -} - -template -inline void write(storage::io::FileWriter &writer, - const detail::PackedVector &vec) -{ - writer.WriteFrom(vec.num_elements); - storage::serialization::write(writer, vec.vec); -} - template inline void read(storage::tar::FileReader &reader, const std::string &name, @@ -86,47 +56,6 @@ inline void write(storage::tar::FileWriter &writer, storage::serialization::write(writer, name + "/packed", vec.vec); } -template -inline void read(storage::io::FileReader &reader, StaticGraph &graph) -{ - storage::serialization::read(reader, graph.node_array); - storage::serialization::read(reader, graph.edge_array); - graph.number_of_nodes = graph.node_array.size() - 1; - graph.number_of_edges = graph.edge_array.size(); -} - -template -inline void write(storage::io::FileWriter &writer, const StaticGraph &graph) -{ - storage::serialization::write(writer, graph.node_array); - storage::serialization::write(writer, graph.edge_array); -} - -template -inline void read(storage::io::FileReader &reader, DynamicGraph &graph) -{ - storage::serialization::read(reader, graph.node_array); - const auto num_edges = reader.ReadElementCount64(); - graph.edge_list.resize(num_edges); - for (auto index : irange(0, num_edges)) - { - reader.ReadInto(graph.edge_list[index]); - } - graph.number_of_nodes = graph.node_array.size(); - graph.number_of_edges = num_edges; -} - -template -inline void write(storage::io::FileWriter &writer, const DynamicGraph &graph) -{ - storage::serialization::write(writer, graph.node_array); - writer.WriteElementCount64(graph.number_of_edges); - for (auto index : irange(0, graph.number_of_edges)) - { - writer.WriteFrom(graph.edge_list[index]); - } -} - template inline void read(storage::tar::FileReader &reader, const std::string &name, diff --git a/include/util/static_graph.hpp b/include/util/static_graph.hpp index 958fdc66f..433a830c1 100644 --- a/include/util/static_graph.hpp +++ b/include/util/static_graph.hpp @@ -8,7 +8,6 @@ #include "util/typedefs.hpp" #include "util/vector_view.hpp" -#include "storage/io_fwd.hpp" #include "storage/shared_memory_ownership.hpp" #include "storage/tar_fwd.hpp" @@ -28,12 +27,6 @@ template class StaticGraph; namespace serialization { -template -void read(storage::io::FileReader &reader, StaticGraph &graph); - -template -void write(storage::io::FileWriter &writer, const StaticGraph &graph); - template void read(storage::tar::FileReader &reader, const std::string &name, @@ -283,12 +276,6 @@ class StaticGraph util::inplacePermutation(edge_array.begin(), edge_array.end(), old_to_new_edge); } - friend void serialization::read(storage::io::FileReader &reader, - StaticGraph &graph); - friend void - serialization::write(storage::io::FileWriter &writer, - const StaticGraph &graph); - friend void serialization::read(storage::tar::FileReader &reader, const std::string &name, StaticGraph &graph); diff --git a/unit_tests/storage/serialization.cpp b/unit_tests/storage/serialization.cpp index 7f4de45e5..26845e8ed 100644 --- a/unit_tests/storage/serialization.cpp +++ b/unit_tests/storage/serialization.cpp @@ -32,26 +32,6 @@ BOOST_AUTO_TEST_CASE(unpack_test) BOOST_CHECK_EQUAL_COLLECTIONS(v.begin(), v.end(), expected.begin(), expected.end()); } -BOOST_AUTO_TEST_CASE(bin_serialize_bool_vector) -{ - TemporaryFile tmp; - { - std::vector> data = { - {}, {0}, {1, 1, 1}, {1, 1, 0, 0, 1, 1, 0, 0}, {1, 1, 0, 0, 1, 1, 0, 0, 1}}; - for (const auto &v : data) - { - { - io::FileWriter writer(tmp.path, io::FileWriter::GenerateFingerprint); - storage::serialization::write(writer, v); - } - std::vector result; - io::FileReader reader(tmp.path, io::FileReader::VerifyFingerprint); - storage::serialization::read(reader, result); - BOOST_CHECK_EQUAL_COLLECTIONS(v.begin(), v.end(), result.begin(), result.end()); - } - } -} - BOOST_AUTO_TEST_CASE(tar_serialize_bool_vector) { TemporaryFile tmp; diff --git a/unit_tests/util/io.cpp b/unit_tests/util/io.cpp index a88b6ad66..1ebeb191d 100644 --- a/unit_tests/util/io.cpp +++ b/unit_tests/util/io.cpp @@ -32,12 +32,14 @@ BOOST_AUTO_TEST_CASE(io_data) { osrm::storage::io::FileWriter outfile(IO_TMP_FILE, osrm::storage::io::FileWriter::GenerateFingerprint); - osrm::storage::serialization::write(outfile, data_in); + outfile.WriteElementCount64(data_in.size()); + outfile.WriteFrom(data_in.data(), data_in.size()); } osrm::storage::io::FileReader infile(IO_TMP_FILE, osrm::storage::io::FileReader::VerifyFingerprint); - osrm::storage::serialization::read(infile, data_out); + data_out.resize(infile.ReadElementCount64()); + infile.ReadInto(data_out.data(), data_out.size()); BOOST_REQUIRE_EQUAL(data_in.size(), data_out.size()); BOOST_CHECK_EQUAL_COLLECTIONS(data_out.begin(), data_out.end(), data_in.begin(), data_in.end()); @@ -88,7 +90,8 @@ BOOST_AUTO_TEST_CASE(file_too_small) { osrm::storage::io::FileWriter outfile( IO_TOO_SMALL_FILE, osrm::storage::io::FileWriter::GenerateFingerprint); - osrm::storage::serialization::write(outfile, v); + outfile.WriteElementCount64(v.size()); + outfile.WriteFrom(v.data(), v.size()); } std::fstream f(IO_TOO_SMALL_FILE); @@ -102,7 +105,8 @@ BOOST_AUTO_TEST_CASE(file_too_small) osrm::storage::io::FileReader infile(IO_TOO_SMALL_FILE, osrm::storage::io::FileReader::VerifyFingerprint); std::vector buffer; - osrm::storage::serialization::read(infile, buffer); + buffer.resize(infile.ReadElementCount64()); + infile.ReadInto(buffer.data(), buffer.size()); BOOST_REQUIRE_MESSAGE(false, "Should not get here"); } catch (const osrm::util::RuntimeError &e) @@ -124,7 +128,8 @@ BOOST_AUTO_TEST_CASE(io_corrupt_fingerprint) osrm::storage::io::FileWriter::HasNoFingerprint); outfile.WriteFrom(0xDEADBEEFCAFEFACE); - osrm::storage::serialization::write(outfile, v); + outfile.WriteElementCount64(v.size()); + outfile.WriteFrom(v.data(), v.size()); } try @@ -155,7 +160,8 @@ BOOST_AUTO_TEST_CASE(io_incompatible_fingerprint) const auto fingerprint = osrm::util::FingerPrint::GetValid(); outfile.WriteFrom(fingerprint); - osrm::storage::serialization::write(outfile, v); + outfile.WriteElementCount64(v.size()); + outfile.WriteFrom(v.data(), v.size()); } std::fstream f(IO_INCOMPATIBLE_FINGERPRINT_FILE);