Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add unit tests for tar reading

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-14 23:42:18 +00:00
parent 86bfe1ede1
commit 2eb633bc41
3 changed files with 142 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
include/storage/tar.hpp
View File

@ -22,43 +22,64 @@ class TarFileReader
public:
TarFileReader(const boost::filesystem::path &path) : path(path)
{
mtar_open(&handle, path.c_str(), "r");
auto ret = mtar_open(&handle, path.c_str(), "r");
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
}
}
~TarFileReader()
{
mtar_close(&handle);
}
template <typename T> T ReadOne(const std::string &name)
{
mtar_header_t header;
mtar_find(&handle, name.c_str(), &header);
if (header.size != sizeof(T))
{
throw util::exception("Datatype size does not match file size.");
}
T tmp;
mtar_read_data(&handle, reinterpret_cast<char *>(&tmp), header.size);
ReadInto(name, &tmp, 1);
return tmp;
}
template <typename T>
void ReadInto(const std::string &name, T &data, const std::size_t number_of_entries)
void ReadInto(const std::string &name, std::vector<T> &data)
{
ReadInto(name, data.data(), data.size());
}
template <typename T>
void ReadInto(const std::string &name, T *data, const std::size_t number_of_entries)
{
mtar_header_t header;
mtar_find(&handle, name.c_str(), &header);
auto ret = mtar_find(&handle, name.c_str(), &header);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
}
if (header.size != sizeof(T) * number_of_entries)
{
throw util::exception("Datatype size does not match file size.");
}
mtar_read_data(&handle, reinterpret_cast<char *>(&data), header.size);
ret = mtar_read_data(&handle, reinterpret_cast<char *>(data), header.size);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
}
}
using TarEntry = std::tuple<std::string, std::size_t>;
template <typename OutIter> void List(OutIter out)
{
mtar_header_t header;
while ((mtar_read_header(&handle, &header)) != MTAR_ENULLRECORD)
while (mtar_read_header(&handle, &header) != MTAR_ENULLRECORD)
{
*out++ = std::tuple<std::string, std::uint64_t>(header.name, header.size);
if (header.type == MTAR_TREG)
{
*out++ = TarEntry {header.name, header.size};
}
mtar_next(&handle);
}
}
@ -97,21 +118,43 @@ class TarFileWriter
public:
TarFileWriter(const boost::filesystem::path &path) : path(path)
{
mtar_open(&handle, path.c_str(), "w");
auto ret = mtar_open(&handle, path.c_str(), "w");
if (ret != MTAR_ESUCCESS)
throw util::exception(mtar_strerror(ret));
WriteFingerprint();
}
~TarFileWriter()
{
mtar_finalize(&handle);
mtar_close(&handle);
}
template <typename T> void WriteOne(const std::string &name, const T &data)
{
mtar_write_file_header(&handle, name.c_str(), name.size());
mtar_write_data(&handle, reinterpret_cast<const char *>(&data), sizeof(T));
WriteFrom(name, &data, 1);
}
template <typename T>
void WriteFrom(const std::string &name, const T &data, const std::size_t number_of_entries)
void WriteFrom(const std::string &name, const std::vector<T> &data)
{
mtar_write_file_header(&handle, name.c_str(), name.size());
mtar_write_data(&handle, reinterpret_cast<const char *>(&data), number_of_entries * sizeof(T));
WriteFrom(name, data.data(), data.size());
}
template <typename T>
void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_entries)
{
auto number_of_bytes = number_of_entries * sizeof(T);
auto ret = mtar_write_file_header(&handle, name.c_str(), number_of_bytes);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
}
ret = mtar_write_data(&handle, reinterpret_cast<const char *>(data), number_of_bytes);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(mtar_strerror(ret));
}
}
private:

BIN
test/data/tar_test.tar Normal file
View File

Binary file not shown.

79
unit_tests/storage/tar.cpp Normal file
View File

@ -0,0 +1,79 @@
#include "storage/tar.hpp"
#include <boost/test/unit_test.hpp>
BOOST_AUTO_TEST_SUITE(tar)
using namespace osrm;
BOOST_AUTO_TEST_CASE(list_tar_file)
{
storage::TarFileReader reader(TEST_DATA_DIR "/tar_test.tar");
std::vector<storage::TarFileReader::TarEntry> file_list;
reader.List(std::back_inserter(file_list));
auto reference_0 = storage::TarFileReader::TarEntry{"foo_1.txt", 4};
auto reference_1 = storage::TarFileReader::TarEntry{"bla/foo_2.txt", 4};
auto reference_2 = storage::TarFileReader::TarEntry{"foo_3.txt", 4};
BOOST_CHECK_EQUAL(std::get<0>(file_list[0]), std::get<0>(reference_0));
BOOST_CHECK_EQUAL(std::get<1>(file_list[0]), std::get<1>(reference_0));
BOOST_CHECK_EQUAL(std::get<0>(file_list[1]), std::get<0>(reference_1));
BOOST_CHECK_EQUAL(std::get<1>(file_list[1]), std::get<1>(reference_1));
BOOST_CHECK_EQUAL(std::get<0>(file_list[2]), std::get<0>(reference_2));
BOOST_CHECK_EQUAL(std::get<1>(file_list[2]), std::get<1>(reference_2));
}
BOOST_AUTO_TEST_CASE(read_tar_file)
{
storage::TarFileReader reader(TEST_DATA_DIR "/tar_test.tar");
char result_0[4];
reader.ReadInto("foo_1.txt", result_0, 4);
// Note: This is an out-of-order read, foo_3 comes after bla/foo_2 in the tar file
char result_1[4];
reader.ReadInto("foo_3.txt", result_1, 4);
char result_2[4];
reader.ReadInto("bla/foo_2.txt", result_2, 4);
BOOST_CHECK_EQUAL(std::string(result_0, 4), std::string("bla\n"));
BOOST_CHECK_EQUAL(std::string(result_1, 4), std::string("foo\n"));
BOOST_CHECK_EQUAL(std::string(result_2, 4), std::string("baz\n"));
}
BOOST_AUTO_TEST_CASE(write_tar_file)
{
boost::filesystem::path tmp_path(TEST_DATA_DIR "/tar_write_test.tar");
std::uint64_t single_64bit_integer = 0xDEADBEEFAABBCCDD;
std::uint32_t single_32bit_integer = 0xDEADBEEF;
std::vector<std::uint32_t> vector_32bit = {0, 1, 2, 3, 4, 1 << 30, 0, 1 << 22, 0xFFFFFFFF};
std::vector<std::uint64_t> vector_64bit = {
0, 1, 2, 3, 4, 1ULL << 62, 0, 1 << 22, 0xFFFFFFFFFFFFFFFF};
{
storage::TarFileWriter writer(tmp_path);
writer.WriteOne("foo/single_64bit_integer", single_64bit_integer);
writer.WriteOne("bar/single_32bit_integer", single_32bit_integer);
writer.WriteFrom("baz/bla/64bit_vector", vector_64bit);
writer.WriteFrom("32bit_vector", vector_32bit);
}
storage::TarFileReader reader(tmp_path);
BOOST_CHECK_EQUAL(reader.ReadOne<std::uint32_t>("bar/single_32bit_integer"),
single_32bit_integer);
BOOST_CHECK_EQUAL(reader.ReadOne<std::uint64_t>("foo/single_64bit_integer"),
single_64bit_integer);
std::vector<std::uint64_t> result_64bit_vector(vector_64bit.size());
reader.ReadInto("baz/bla/64bit_vector", result_64bit_vector);
std::vector<std::uint32_t> result_32bit_vector(vector_32bit.size());
reader.ReadInto("32bit_vector", result_32bit_vector);
}
BOOST_AUTO_TEST_SUITE_END()