From 495131efd7d25461e40bf96316775f9999f6e0de Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 15 Mar 2018 00:27:16 +0000 Subject: [PATCH] Add vector serialization --- include/storage/serialization.hpp | 29 +++++++++++++++++++++++ include/storage/tar.hpp | 38 ++++++++++++++++--------------- unit_tests/storage/tar.cpp | 36 +++++++++++++++++------------ 3 files changed, 71 insertions(+), 32 deletions(-) diff --git a/include/storage/serialization.hpp b/include/storage/serialization.hpp index 2afb835ac..e542e4559 100644 --- a/include/storage/serialization.hpp +++ b/include/storage/serialization.hpp @@ -6,6 +6,7 @@ #include "util/vector_view.hpp" #include "storage/io.hpp" +#include "storage/tar.hpp" #include #include @@ -86,6 +87,34 @@ inline void write(storage::io::FileWriter &writer, const stxxl::vector &vec) } #endif +template void read(tar::FileReader &reader, const std::string& name, std::vector &data) +{ + const auto count = reader.ReadElementCount64(name); + data.resize(count); + reader.ReadInto(name, data.data(), count); +} + +template void write(tar::FileWriter &writer, const std::string& name, const std::vector &data) +{ + const auto count = data.size(); + writer.WriteElementCount64(name, count); + writer.WriteFrom(name, data.data(), count); +} + +template void read(tar::FileReader &reader, const std::string& name, util::vector_view &data) +{ + const auto count = reader.ReadElementCount64(name); + BOOST_ASSERT(data.size() == count); + reader.ReadInto(name, data.data(), count); +} + +template void write(tar::FileWriter &writer, const std::string& name, const util::vector_view &data) +{ + const auto count = data.size(); + writer.WriteElementCount64(name, count); + writer.WriteFrom(name, data.data(), count); +} + template void read(io::FileReader &reader, std::vector &data) { const auto count = reader.ReadElementCount64(); diff --git a/include/storage/tar.hpp b/include/storage/tar.hpp index 611f558ba..5af2514a8 100644 --- a/include/storage/tar.hpp +++ b/include/storage/tar.hpp @@ -16,11 +16,13 @@ namespace osrm { namespace storage { +namespace tar +{ -class TarFileReader +class FileReader { public: - TarFileReader(const boost::filesystem::path &path) : path(path) + FileReader(const boost::filesystem::path &path) : path(path) { auto ret = mtar_open(&handle, path.c_str(), "r"); if (ret != MTAR_ESUCCESS) @@ -29,11 +31,16 @@ class TarFileReader } } - ~TarFileReader() + ~FileReader() { mtar_close(&handle); } + std::uint64_t ReadElementCount64(const std::string &name) + { + return ReadOne(name + "_count"); + } + template T ReadOne(const std::string &name) { T tmp; @@ -41,12 +48,6 @@ class TarFileReader return tmp; } - template - void ReadInto(const std::string &name, std::vector &data) - { - ReadInto(name, data.data(), data.size()); - } - template void ReadInto(const std::string &name, T *data, const std::size_t number_of_entries) { @@ -113,10 +114,10 @@ class TarFileReader mtar_t handle; }; -class TarFileWriter +class FileWriter { public: - TarFileWriter(const boost::filesystem::path &path) : path(path) + FileWriter(const boost::filesystem::path &path) : path(path) { auto ret = mtar_open(&handle, path.c_str(), "w"); if (ret != MTAR_ESUCCESS) @@ -124,23 +125,22 @@ class TarFileWriter WriteFingerprint(); } - ~TarFileWriter() + ~FileWriter() { mtar_finalize(&handle); mtar_close(&handle); } + void WriteElementCount64(const std::string &name, const std::uint64_t count) + { + WriteOne(name + "_count", count); + } + template void WriteOne(const std::string &name, const T &data) { WriteFrom(name, &data, 1); } - template - void WriteFrom(const std::string &name, const std::vector &data) - { - WriteFrom(name, data.data(), data.size()); - } - template void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_entries) { @@ -167,6 +167,8 @@ class TarFileWriter boost::filesystem::path path; mtar_t handle; }; + +} } } diff --git a/unit_tests/storage/tar.cpp b/unit_tests/storage/tar.cpp index a17284adc..b4cfe9cde 100644 --- a/unit_tests/storage/tar.cpp +++ b/unit_tests/storage/tar.cpp @@ -1,5 +1,7 @@ #include "storage/tar.hpp" +#include "../common/range_tools.hpp" + #include BOOST_AUTO_TEST_SUITE(tar) @@ -8,14 +10,14 @@ using namespace osrm; BOOST_AUTO_TEST_CASE(list_tar_file) { - storage::TarFileReader reader(TEST_DATA_DIR "/tar_test.tar"); + storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar"); - std::vector file_list; + std::vector file_list; reader.List(std::back_inserter(file_list)); - auto reference_0 = storage::TarFileReader::TarEntry{"foo_1.txt", 4}; - auto reference_1 = storage::TarFileReader::TarEntry{"bla/foo_2.txt", 4}; - auto reference_2 = storage::TarFileReader::TarEntry{"foo_3.txt", 4}; + auto reference_0 = storage::tar::FileReader::TarEntry{"foo_1.txt", 4}; + auto reference_1 = storage::tar::FileReader::TarEntry{"bla/foo_2.txt", 4}; + auto reference_2 = storage::tar::FileReader::TarEntry{"foo_3.txt", 4}; BOOST_CHECK_EQUAL(std::get<0>(file_list[0]), std::get<0>(reference_0)); BOOST_CHECK_EQUAL(std::get<1>(file_list[0]), std::get<1>(reference_0)); @@ -27,7 +29,7 @@ BOOST_AUTO_TEST_CASE(list_tar_file) BOOST_AUTO_TEST_CASE(read_tar_file) { - storage::TarFileReader reader(TEST_DATA_DIR "/tar_test.tar"); + storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar"); char result_0[4]; reader.ReadInto("foo_1.txt", result_0, 4); @@ -56,24 +58,30 @@ BOOST_AUTO_TEST_CASE(write_tar_file) 0, 1, 2, 3, 4, 1ULL << 62, 0, 1 << 22, 0xFFFFFFFFFFFFFFFF}; { - storage::TarFileWriter writer(tmp_path); + storage::tar::FileWriter writer(tmp_path); writer.WriteOne("foo/single_64bit_integer", single_64bit_integer); writer.WriteOne("bar/single_32bit_integer", single_32bit_integer); - writer.WriteFrom("baz/bla/64bit_vector", vector_64bit); - writer.WriteFrom("32bit_vector", vector_32bit); + writer.WriteElementCount64("baz/bla/64bit_vector", vector_64bit.size()); + writer.WriteFrom("baz/bla/64bit_vector", vector_64bit.data(), vector_64bit.size()); + writer.WriteElementCount64("32bit_vector", vector_32bit.size()); + writer.WriteFrom("32bit_vector", vector_32bit.data(), vector_32bit.size()); } - storage::TarFileReader reader(tmp_path); + storage::tar::FileReader reader(tmp_path); BOOST_CHECK_EQUAL(reader.ReadOne("bar/single_32bit_integer"), single_32bit_integer); BOOST_CHECK_EQUAL(reader.ReadOne("foo/single_64bit_integer"), single_64bit_integer); - std::vector result_64bit_vector(vector_64bit.size()); - reader.ReadInto("baz/bla/64bit_vector", result_64bit_vector); - std::vector result_32bit_vector(vector_32bit.size()); - reader.ReadInto("32bit_vector", result_32bit_vector); + std::vector result_64bit_vector( + reader.ReadElementCount64("baz/bla/64bit_vector")); + reader.ReadInto("baz/bla/64bit_vector", result_64bit_vector.data(), result_64bit_vector.size()); + std::vector result_32bit_vector(reader.ReadElementCount64("32bit_vector")); + reader.ReadInto("32bit_vector", result_32bit_vector.data(), result_32bit_vector.size()); + + CHECK_EQUAL_COLLECTIONS(result_64bit_vector, vector_64bit); + CHECK_EQUAL_COLLECTIONS(result_32bit_vector, vector_32bit); } BOOST_AUTO_TEST_SUITE_END()