Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add vector serialization

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-15 00:27:16 +00:00
parent 2eb633bc41
commit 495131efd7
3 changed files with 71 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
include/storage/serialization.hpp
View File

@ -6,6 +6,7 @@
#include "util/vector_view.hpp"
#include "storage/io.hpp"
#include "storage/tar.hpp"
#include <cmath>
#include <cstdint>
@ -86,6 +87,34 @@ inline void write(storage::io::FileWriter &writer, const stxxl::vector<T> &vec)
}
#endif
template <typename T> void read(tar::FileReader &reader, const std::string& name, std::vector<T> &data)
{
const auto count = reader.ReadElementCount64(name);
data.resize(count);
reader.ReadInto(name, data.data(), count);
}
template <typename T> void write(tar::FileWriter &writer, const std::string& name, const std::vector<T> &data)
{
const auto count = data.size();
writer.WriteElementCount64(name, count);
writer.WriteFrom(name, data.data(), count);
}
template <typename T> void read(tar::FileReader &reader, const std::string& name, util::vector_view<T> &data)
{
const auto count = reader.ReadElementCount64(name);
BOOST_ASSERT(data.size() == count);
reader.ReadInto(name, data.data(), count);
}
template <typename T> void write(tar::FileWriter &writer, const std::string& name, const util::vector_view<T> &data)
{
const auto count = data.size();
writer.WriteElementCount64(name, count);
writer.WriteFrom(name, data.data(), count);
}
template <typename T> void read(io::FileReader &reader, std::vector<T> &data)
{
const auto count = reader.ReadElementCount64();

38
include/storage/tar.hpp
View File

@ -16,11 +16,13 @@ namespace osrm
{
namespace storage
{
namespace tar
{
class TarFileReader
class FileReader
{
public:
TarFileReader(const boost::filesystem::path &path) : path(path)
FileReader(const boost::filesystem::path &path) : path(path)
{
auto ret = mtar_open(&handle, path.c_str(), "r");
if (ret != MTAR_ESUCCESS)
@ -29,11 +31,16 @@ class TarFileReader
}
}
~TarFileReader()
~FileReader()
{
mtar_close(&handle);
}
std::uint64_t ReadElementCount64(const std::string &name)
{
return ReadOne<std::uint64_t>(name + "_count");
}
template <typename T> T ReadOne(const std::string &name)
{
T tmp;
@ -41,12 +48,6 @@ class TarFileReader
return tmp;
}
template <typename T>
void ReadInto(const std::string &name, std::vector<T> &data)
{
ReadInto(name, data.data(), data.size());
}
template <typename T>
void ReadInto(const std::string &name, T *data, const std::size_t number_of_entries)
{
@ -113,10 +114,10 @@ class TarFileReader
mtar_t handle;
};
class TarFileWriter
class FileWriter
{
public:
TarFileWriter(const boost::filesystem::path &path) : path(path)
FileWriter(const boost::filesystem::path &path) : path(path)
{
auto ret = mtar_open(&handle, path.c_str(), "w");
if (ret != MTAR_ESUCCESS)
@ -124,23 +125,22 @@ class TarFileWriter
WriteFingerprint();
}
~TarFileWriter()
~FileWriter()
{
mtar_finalize(&handle);
mtar_close(&handle);
}
void WriteElementCount64(const std::string &name, const std::uint64_t count)
{
WriteOne(name + "_count", count);
}
template <typename T> void WriteOne(const std::string &name, const T &data)
{
WriteFrom(name, &data, 1);
}
template <typename T>
void WriteFrom(const std::string &name, const std::vector<T> &data)
{
WriteFrom(name, data.data(), data.size());
}
template <typename T>
void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_entries)
{
@ -167,6 +167,8 @@ class TarFileWriter
boost::filesystem::path path;
mtar_t handle;
};
}
}
}

36
unit_tests/storage/tar.cpp
View File

@ -1,5 +1,7 @@
#include "storage/tar.hpp"
#include "../common/range_tools.hpp"
#include <boost/test/unit_test.hpp>
BOOST_AUTO_TEST_SUITE(tar)
@ -8,14 +10,14 @@ using namespace osrm;
BOOST_AUTO_TEST_CASE(list_tar_file)
{
storage::TarFileReader reader(TEST_DATA_DIR "/tar_test.tar");
storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar");
std::vector<storage::TarFileReader::TarEntry> file_list;
std::vector<storage::tar::FileReader::TarEntry> file_list;
reader.List(std::back_inserter(file_list));
auto reference_0 = storage::TarFileReader::TarEntry{"foo_1.txt", 4};
auto reference_1 = storage::TarFileReader::TarEntry{"bla/foo_2.txt", 4};
auto reference_2 = storage::TarFileReader::TarEntry{"foo_3.txt", 4};
auto reference_0 = storage::tar::FileReader::TarEntry{"foo_1.txt", 4};
auto reference_1 = storage::tar::FileReader::TarEntry{"bla/foo_2.txt", 4};
auto reference_2 = storage::tar::FileReader::TarEntry{"foo_3.txt", 4};
BOOST_CHECK_EQUAL(std::get<0>(file_list[0]), std::get<0>(reference_0));
BOOST_CHECK_EQUAL(std::get<1>(file_list[0]), std::get<1>(reference_0));
@ -27,7 +29,7 @@ BOOST_AUTO_TEST_CASE(list_tar_file)
BOOST_AUTO_TEST_CASE(read_tar_file)
{
storage::TarFileReader reader(TEST_DATA_DIR "/tar_test.tar");
storage::tar::FileReader reader(TEST_DATA_DIR "/tar_test.tar");
char result_0[4];
reader.ReadInto("foo_1.txt", result_0, 4);
@ -56,24 +58,30 @@ BOOST_AUTO_TEST_CASE(write_tar_file)
0, 1, 2, 3, 4, 1ULL << 62, 0, 1 << 22, 0xFFFFFFFFFFFFFFFF};
{
storage::TarFileWriter writer(tmp_path);
storage::tar::FileWriter writer(tmp_path);
writer.WriteOne("foo/single_64bit_integer", single_64bit_integer);
writer.WriteOne("bar/single_32bit_integer", single_32bit_integer);
writer.WriteFrom("baz/bla/64bit_vector", vector_64bit);
writer.WriteFrom("32bit_vector", vector_32bit);
writer.WriteElementCount64("baz/bla/64bit_vector", vector_64bit.size());
writer.WriteFrom("baz/bla/64bit_vector", vector_64bit.data(), vector_64bit.size());
writer.WriteElementCount64("32bit_vector", vector_32bit.size());
writer.WriteFrom("32bit_vector", vector_32bit.data(), vector_32bit.size());
}
storage::TarFileReader reader(tmp_path);
storage::tar::FileReader reader(tmp_path);
BOOST_CHECK_EQUAL(reader.ReadOne<std::uint32_t>("bar/single_32bit_integer"),
single_32bit_integer);
BOOST_CHECK_EQUAL(reader.ReadOne<std::uint64_t>("foo/single_64bit_integer"),
single_64bit_integer);
std::vector<std::uint64_t> result_64bit_vector(vector_64bit.size());
reader.ReadInto("baz/bla/64bit_vector", result_64bit_vector);
std::vector<std::uint32_t> result_32bit_vector(vector_32bit.size());
reader.ReadInto("32bit_vector", result_32bit_vector);
std::vector<std::uint64_t> result_64bit_vector(
reader.ReadElementCount64("baz/bla/64bit_vector"));
reader.ReadInto("baz/bla/64bit_vector", result_64bit_vector.data(), result_64bit_vector.size());
std::vector<std::uint32_t> result_32bit_vector(reader.ReadElementCount64("32bit_vector"));
reader.ReadInto("32bit_vector", result_32bit_vector.data(), result_32bit_vector.size());
CHECK_EQUAL_COLLECTIONS(result_64bit_vector, vector_64bit);
CHECK_EQUAL_COLLECTIONS(result_32bit_vector, vector_32bit);
}
BOOST_AUTO_TEST_SUITE_END()