Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add function to stop and continue writing to a tar file

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-22 11:33:02 +00:00
parent e25654c210
commit 8e800c48bc
2 changed files with 65 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
include/storage/tar.hpp
View File

@ -215,6 +215,39 @@ class FileWriter
} }
} }
// Continue writing an existing file, overwrites all data after the file!
template <typename T>
void ContinueFrom(const std::string &name, const T *data, const std::size_t number_of_elements)
{
auto number_of_bytes = number_of_elements * sizeof(T);
mtar_header_t header;
auto ret = mtar_find(&handle, name.c_str(), &header);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(name + ": Error reading header: " + mtar_strerror(ret));
}
// update header to reflect increased tar size
auto old_size = header.size;
header.size += number_of_bytes;
mtar_write_header(&handle, &header);
// now seek to the end of the old record
handle.remaining_data = number_of_bytes;
ret = mtar_seek(&handle, handle.pos + old_size);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(name + ": Error seeking to end of old data: " + mtar_strerror(ret));
}
ret = mtar_write_data(&handle, data, number_of_bytes);
if (ret != MTAR_ESUCCESS)
{
throw util::exception(name + ": Error writing data : " + mtar_strerror(ret));
}
}
template <typename T> template <typename T>
void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_elements) void WriteFrom(const std::string &name, const T *data, const std::size_t number_of_elements)
{ {

33
unit_tests/storage/tar.cpp
View File

@ -77,7 +77,7 @@ BOOST_AUTO_TEST_CASE(write_tar_file)
reader.ReadInto("bar/single_32bit_integer", result_32bit_integer); reader.ReadInto("bar/single_32bit_integer", result_32bit_integer);
reader.ReadInto("foo/single_64bit_integer", result_64bit_integer); reader.ReadInto("foo/single_64bit_integer", result_64bit_integer);
BOOST_CHECK_EQUAL(result_32bit_integer, single_32bit_integer); BOOST_CHECK_EQUAL(result_32bit_integer, single_32bit_integer);
BOOST_CHECK_EQUAL(result_32bit_integer, single_64bit_integer); BOOST_CHECK_EQUAL(result_64bit_integer, single_64bit_integer);
std::vector<std::uint64_t> result_64bit_vector( std::vector<std::uint64_t> result_64bit_vector(
reader.ReadElementCount64("baz/bla/64bit_vector")); reader.ReadElementCount64("baz/bla/64bit_vector"));
@ -89,4 +89,35 @@ BOOST_AUTO_TEST_CASE(write_tar_file)
CHECK_EQUAL_COLLECTIONS(result_32bit_vector, vector_32bit); CHECK_EQUAL_COLLECTIONS(result_32bit_vector, vector_32bit);
} }
BOOST_AUTO_TEST_CASE(continue_write_tar_file)
{
TemporaryFile tmp{TEST_DATA_DIR "/tar_continue_write_test.tar"};
// more than 64 values to ensure we fill up more than one tar block of 512 bytes
std::vector<std::uint64_t> vector_64bit = {
0, 1, 2, 3, 4, 1ULL << 62, 0, 1 << 22, 0xFFFFFFFFFFFFFFFF, 0xFF00FF0000FF00FF, 11, 12, 13, 14, 15, 16,
17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48,
49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64,
65, 66, 67, 68, 69, 70
};
{
storage::tar::FileWriter writer(tmp.path, storage::tar::FileWriter::GenerateFingerprint);
writer.WriteElementCount64("baz/bla/64bit_vector", vector_64bit.size());
writer.WriteFrom("baz/bla/64bit_vector", vector_64bit.data(), 12);
writer.ContinueFrom("baz/bla/64bit_vector", vector_64bit.data() + 12, 30);
writer.ContinueFrom("baz/bla/64bit_vector", vector_64bit.data() + 42, 10);
writer.ContinueFrom("baz/bla/64bit_vector", vector_64bit.data() + 52, vector_64bit.size() - 52);
}
storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
std::vector<std::uint64_t> result_64bit_vector(
reader.ReadElementCount64("baz/bla/64bit_vector"));
reader.ReadInto("baz/bla/64bit_vector", result_64bit_vector.data(), result_64bit_vector.size());
CHECK_EQUAL_COLLECTIONS(result_64bit_vector, vector_64bit);
}
BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END()