From 2eb633bc4116ee8f5bc319418d5b90f5c9ce210f Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 14 Mar 2018 23:42:18 +0000 Subject: [PATCH] Add unit tests for tar reading --- include/storage/tar.hpp | 83 ++++++++++++++++++++++++++++--------- test/data/tar_test.tar | Bin 0 -> 10240 bytes unit_tests/storage/tar.cpp | 79 +++++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 20 deletions(-) create mode 100644 test/data/tar_test.tar create mode 100644 unit_tests/storage/tar.cpp diff --git a/include/storage/tar.hpp b/include/storage/tar.hpp index 8a3ce92e7..611f558ba 100644 --- a/include/storage/tar.hpp +++ b/include/storage/tar.hpp @@ -22,43 +22,64 @@ class TarFileReader public: TarFileReader(const boost::filesystem::path &path) : path(path) { - mtar_open(&handle, path.c_str(), "r"); + auto ret = mtar_open(&handle, path.c_str(), "r"); + if (ret != MTAR_ESUCCESS) + { + throw util::exception(mtar_strerror(ret)); + } + } + + ~TarFileReader() + { + mtar_close(&handle); } template T ReadOne(const std::string &name) { - mtar_header_t header; - mtar_find(&handle, name.c_str(), &header); - if (header.size != sizeof(T)) - { - throw util::exception("Datatype size does not match file size."); - } - T tmp; - mtar_read_data(&handle, reinterpret_cast(&tmp), header.size); + ReadInto(name, &tmp, 1); return tmp; } template - void ReadInto(const std::string &name, T &data, const std::size_t number_of_entries) + void ReadInto(const std::string &name, std::vector &data) + { + ReadInto(name, data.data(), data.size()); + } + + template + void ReadInto(const std::string &name, T *data, const std::size_t number_of_entries) { mtar_header_t header; - mtar_find(&handle, name.c_str(), &header); + auto ret = mtar_find(&handle, name.c_str(), &header); + if (ret != MTAR_ESUCCESS) + { + throw util::exception(mtar_strerror(ret)); + } + if (header.size != sizeof(T) * number_of_entries) { throw util::exception("Datatype size does not match file size."); } - mtar_read_data(&handle, reinterpret_cast(&data), header.size); + ret = mtar_read_data(&handle, reinterpret_cast(data), header.size); + if (ret != MTAR_ESUCCESS) + { + throw util::exception(mtar_strerror(ret)); + } } using TarEntry = std::tuple; template void List(OutIter out) { mtar_header_t header; - while ((mtar_read_header(&handle, &header)) != MTAR_ENULLRECORD) + while (mtar_read_header(&handle, &header) != MTAR_ENULLRECORD) { - *out++ = std::tuple(header.name, header.size); + if (header.type == MTAR_TREG) + { + *out++ = TarEntry {header.name, header.size}; + } + mtar_next(&handle); } } @@ -97,21 +118,43 @@ class TarFileWriter public: TarFileWriter(const boost::filesystem::path &path) : path(path) { - mtar_open(&handle, path.c_str(), "w"); + auto ret = mtar_open(&handle, path.c_str(), "w"); + if (ret != MTAR_ESUCCESS) + throw util::exception(mtar_strerror(ret)); WriteFingerprint(); } + ~TarFileWriter() + { + mtar_finalize(&handle); + mtar_close(&handle); + } + template void WriteOne(const std::string &name, const T &data) { - mtar_write_file_header(&handle, name.c_str(), name.size()); - mtar_write_data(&handle, reinterpret_cast(&data), sizeof(T)); + WriteFrom(name, &data, 1); } template - void WriteFrom(const std::string &name, const T &data, const std::size_t number_of_entries) + void WriteFrom(const std::string &name, const std::vector &data) { - mtar_write_file_header(&handle, name.c_str(), name.size()); - mtar_write_data(&handle, reinterpret_cast(&data), number_of_entries * sizeof(T)); + WriteFrom(name, data.data(), data.size()); + } + + template + void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_entries) + { + auto number_of_bytes = number_of_entries * sizeof(T); + auto ret = mtar_write_file_header(&handle, name.c_str(), number_of_bytes); + if (ret != MTAR_ESUCCESS) + { + throw util::exception(mtar_strerror(ret)); + } + ret = mtar_write_data(&handle, reinterpret_cast(data), number_of_bytes); + if (ret != MTAR_ESUCCESS) + { + throw util::exception(mtar_strerror(ret)); + } } private: diff --git a/test/data/tar_test.tar b/test/data/tar_test.tar new file mode 100644 index 0000000000000000000000000000000000000000..cb50fcc06bc8106ff58f2feb0ad0c03dac42f72e GIT binary patch literal 10240 zcmeI!(F(#K7=~ewyNX>w>91~=6D}A;L9q;?t51&>+d&0$Fna$Jw>g|V?i0JqWpB3e z5mmX;scPp`YE0wxdz-sbIc2PP5ntD0ls4iTrz%};O2f$aTMT_wscVS+@o*YCMpvKu zN3XBtwmpZ + +BOOST_AUTO_TEST_SUITE(tar) + +using namespace osrm; + +BOOST_AUTO_TEST_CASE(list_tar_file) +{ + storage::TarFileReader reader(TEST_DATA_DIR "/tar_test.tar"); + + std::vector file_list; + reader.List(std::back_inserter(file_list)); + + auto reference_0 = storage::TarFileReader::TarEntry{"foo_1.txt", 4}; + auto reference_1 = storage::TarFileReader::TarEntry{"bla/foo_2.txt", 4}; + auto reference_2 = storage::TarFileReader::TarEntry{"foo_3.txt", 4}; + + BOOST_CHECK_EQUAL(std::get<0>(file_list[0]), std::get<0>(reference_0)); + BOOST_CHECK_EQUAL(std::get<1>(file_list[0]), std::get<1>(reference_0)); + BOOST_CHECK_EQUAL(std::get<0>(file_list[1]), std::get<0>(reference_1)); + BOOST_CHECK_EQUAL(std::get<1>(file_list[1]), std::get<1>(reference_1)); + BOOST_CHECK_EQUAL(std::get<0>(file_list[2]), std::get<0>(reference_2)); + BOOST_CHECK_EQUAL(std::get<1>(file_list[2]), std::get<1>(reference_2)); +} + +BOOST_AUTO_TEST_CASE(read_tar_file) +{ + storage::TarFileReader reader(TEST_DATA_DIR "/tar_test.tar"); + + char result_0[4]; + reader.ReadInto("foo_1.txt", result_0, 4); + + // Note: This is an out-of-order read, foo_3 comes after bla/foo_2 in the tar file + char result_1[4]; + reader.ReadInto("foo_3.txt", result_1, 4); + + char result_2[4]; + reader.ReadInto("bla/foo_2.txt", result_2, 4); + + BOOST_CHECK_EQUAL(std::string(result_0, 4), std::string("bla\n")); + BOOST_CHECK_EQUAL(std::string(result_1, 4), std::string("foo\n")); + BOOST_CHECK_EQUAL(std::string(result_2, 4), std::string("baz\n")); +} + +BOOST_AUTO_TEST_CASE(write_tar_file) +{ + boost::filesystem::path tmp_path(TEST_DATA_DIR "/tar_write_test.tar"); + + std::uint64_t single_64bit_integer = 0xDEADBEEFAABBCCDD; + std::uint32_t single_32bit_integer = 0xDEADBEEF; + + std::vector vector_32bit = {0, 1, 2, 3, 4, 1 << 30, 0, 1 << 22, 0xFFFFFFFF}; + std::vector vector_64bit = { + 0, 1, 2, 3, 4, 1ULL << 62, 0, 1 << 22, 0xFFFFFFFFFFFFFFFF}; + + { + storage::TarFileWriter writer(tmp_path); + writer.WriteOne("foo/single_64bit_integer", single_64bit_integer); + writer.WriteOne("bar/single_32bit_integer", single_32bit_integer); + writer.WriteFrom("baz/bla/64bit_vector", vector_64bit); + writer.WriteFrom("32bit_vector", vector_32bit); + } + + storage::TarFileReader reader(tmp_path); + + BOOST_CHECK_EQUAL(reader.ReadOne("bar/single_32bit_integer"), + single_32bit_integer); + BOOST_CHECK_EQUAL(reader.ReadOne("foo/single_64bit_integer"), + single_64bit_integer); + + std::vector result_64bit_vector(vector_64bit.size()); + reader.ReadInto("baz/bla/64bit_vector", result_64bit_vector); + std::vector result_32bit_vector(vector_32bit.size()); + reader.ReadInto("32bit_vector", result_32bit_vector); +} + +BOOST_AUTO_TEST_SUITE_END()