Add vector serialization

include/storage/serialization.hpp

@@ -6,6 +6,7 @@
 #include "util/vector_view.hpp"
 
 #include "storage/io.hpp"
+#include "storage/tar.hpp"
 
 #include <cmath>
 #include <cstdint>
@@ -86,6 +87,34 @@ inline void write(storage::io::FileWriter &writer, const stxxl::vector<T> &vec)
 }
 #endif
 
+template <typename T> void read(tar::FileReader &reader, const std::string& name, std::vector<T> &data)
+{
+    const auto count = reader.ReadElementCount64(name);
+    data.resize(count);
+    reader.ReadInto(name, data.data(), count);
+}
+
+template <typename T> void write(tar::FileWriter &writer, const std::string& name, const std::vector<T> &data)
+{
+    const auto count = data.size();
+    writer.WriteElementCount64(name, count);
+    writer.WriteFrom(name, data.data(), count);
+}
+
+template <typename T> void read(tar::FileReader &reader, const std::string& name, util::vector_view<T> &data)
+{
+    const auto count = reader.ReadElementCount64(name);
+    BOOST_ASSERT(data.size() == count);
+    reader.ReadInto(name, data.data(), count);
+}
+
+template <typename T> void write(tar::FileWriter &writer, const std::string& name, const util::vector_view<T> &data)
+{
+    const auto count = data.size();
+    writer.WriteElementCount64(name, count);
+    writer.WriteFrom(name, data.data(), count);
+}
+
 template <typename T> void read(io::FileReader &reader, std::vector<T> &data)
 {
     const auto count = reader.ReadElementCount64();

include/storage/tar.hpp

@@ -16,11 +16,13 @@ namespace osrm
 {
 namespace storage
 {
+namespace tar
+{
 
-class TarFileReader
+class FileReader
 {
   public:
-    TarFileReader(const boost::filesystem::path &path) : path(path)
+    FileReader(const boost::filesystem::path &path) : path(path)
     {
         auto ret = mtar_open(&handle, path.c_str(), "r");
         if (ret != MTAR_ESUCCESS)
@@ -29,11 +31,16 @@ class TarFileReader
         }
     }
 
-    ~TarFileReader()
+    ~FileReader()
     {
         mtar_close(&handle);
     }
 
+    std::uint64_t ReadElementCount64(const std::string &name)
+    {
+        return ReadOne<std::uint64_t>(name + "_count");
+    }
+
     template <typename T> T ReadOne(const std::string &name)
     {
         T tmp;
@@ -41,12 +48,6 @@ class TarFileReader
         return tmp;
     }
 
-    template <typename T>
-    void ReadInto(const std::string &name, std::vector<T> &data)
-    {
-        ReadInto(name, data.data(), data.size());
-    }
-
     template <typename T>
     void ReadInto(const std::string &name, T *data, const std::size_t number_of_entries)
     {
@@ -113,10 +114,10 @@ class TarFileReader
     mtar_t handle;
 };
 
-class TarFileWriter
+class FileWriter
 {
   public:
-    TarFileWriter(const boost::filesystem::path &path) : path(path)
+    FileWriter(const boost::filesystem::path &path) : path(path)
     {
         auto ret = mtar_open(&handle, path.c_str(), "w");
         if (ret != MTAR_ESUCCESS)
@@ -124,23 +125,22 @@ class TarFileWriter
         WriteFingerprint();
     }
 
-    ~TarFileWriter()
+    ~FileWriter()
     {
         mtar_finalize(&handle);
         mtar_close(&handle);
     }
 
+    void WriteElementCount64(const std::string &name, const std::uint64_t count)
+    {
+        WriteOne(name + "_count", count);
+    }
+
     template <typename T> void WriteOne(const std::string &name, const T &data)
     {
         WriteFrom(name, &data, 1);
     }
 
-    template <typename T>
-    void WriteFrom(const std::string &name, const std::vector<T> &data)
-    {
-        WriteFrom(name, data.data(), data.size());
-    }
-
     template <typename T>
     void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_entries)
     {
@@ -167,6 +167,8 @@ class TarFileWriter
     boost::filesystem::path path;
     mtar_t handle;
 };
+
+}
 }
 }
 

unit_tests/storage/tar.cpp

@@ -1,5 +1,7 @@
 #include "storage/tar.hpp"
 
+#include "../common/range_tools.hpp"
+
 #include <boost/test/unit_test.hpp>
 
 BOOST_AUTO_TEST_SUITE(tar)
@@ -8,14 +10,14 @@ using namespace osrm;
 
 BOOST_AUTO_TEST_CASE(list_tar_file)
 {
-    storage::TarFileReader reader(TEST_DATA_DIR "/tar_test.tar");
+    storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar");
 
-    std::vector<storage::TarFileReader::TarEntry> file_list;
+    std::vector<storage::tar::FileReader::TarEntry> file_list;
     reader.List(std::back_inserter(file_list));
 
-    auto reference_0 = storage::TarFileReader::TarEntry{"foo_1.txt", 4};
+    auto reference_0 = storage::tar::FileReader::TarEntry{"foo_1.txt", 4};
-    auto reference_1 = storage::TarFileReader::TarEntry{"bla/foo_2.txt", 4};
+    auto reference_1 = storage::tar::FileReader::TarEntry{"bla/foo_2.txt", 4};
-    auto reference_2 = storage::TarFileReader::TarEntry{"foo_3.txt", 4};
+    auto reference_2 = storage::tar::FileReader::TarEntry{"foo_3.txt", 4};
 
     BOOST_CHECK_EQUAL(std::get<0>(file_list[0]), std::get<0>(reference_0));
     BOOST_CHECK_EQUAL(std::get<1>(file_list[0]), std::get<1>(reference_0));
@@ -27,7 +29,7 @@ BOOST_AUTO_TEST_CASE(list_tar_file)
 
 BOOST_AUTO_TEST_CASE(read_tar_file)
 {
-    storage::TarFileReader reader(TEST_DATA_DIR "/tar_test.tar");
+    storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar");
 
     char result_0[4];
     reader.ReadInto("foo_1.txt", result_0, 4);
@@ -56,24 +58,30 @@ BOOST_AUTO_TEST_CASE(write_tar_file)
         0, 1, 2, 3, 4, 1ULL << 62, 0, 1 << 22, 0xFFFFFFFFFFFFFFFF};
 
     {
-        storage::TarFileWriter writer(tmp_path);
+        storage::tar::FileWriter writer(tmp_path);
         writer.WriteOne("foo/single_64bit_integer", single_64bit_integer);
         writer.WriteOne("bar/single_32bit_integer", single_32bit_integer);
-        writer.WriteFrom("baz/bla/64bit_vector", vector_64bit);
+        writer.WriteElementCount64("baz/bla/64bit_vector", vector_64bit.size());
-        writer.WriteFrom("32bit_vector", vector_32bit);
+        writer.WriteFrom("baz/bla/64bit_vector", vector_64bit.data(), vector_64bit.size());
+        writer.WriteElementCount64("32bit_vector", vector_32bit.size());
+        writer.WriteFrom("32bit_vector", vector_32bit.data(), vector_32bit.size());
     }
 
-    storage::TarFileReader reader(tmp_path);
+    storage::tar::FileReader reader(tmp_path);
 
     BOOST_CHECK_EQUAL(reader.ReadOne<std::uint32_t>("bar/single_32bit_integer"),
                       single_32bit_integer);
     BOOST_CHECK_EQUAL(reader.ReadOne<std::uint64_t>("foo/single_64bit_integer"),
                       single_64bit_integer);
 
-    std::vector<std::uint64_t> result_64bit_vector(vector_64bit.size());
+    std::vector<std::uint64_t> result_64bit_vector(
-    reader.ReadInto("baz/bla/64bit_vector", result_64bit_vector);
+        reader.ReadElementCount64("baz/bla/64bit_vector"));
-    std::vector<std::uint32_t> result_32bit_vector(vector_32bit.size());
+    reader.ReadInto("baz/bla/64bit_vector", result_64bit_vector.data(), result_64bit_vector.size());
-    reader.ReadInto("32bit_vector", result_32bit_vector);
+    std::vector<std::uint32_t> result_32bit_vector(reader.ReadElementCount64("32bit_vector"));
+    reader.ReadInto("32bit_vector", result_32bit_vector.data(), result_32bit_vector.size());
+
+    CHECK_EQUAL_COLLECTIONS(result_64bit_vector, vector_64bit);
+    CHECK_EQUAL_COLLECTIONS(result_32bit_vector, vector_32bit);
 }
 
 BOOST_AUTO_TEST_SUITE_END()