From fb02a4c674fcfb1530568e67aab027c4a562e571 Mon Sep 17 00:00:00 2001 From: Michael Krasnyk Date: Thu, 17 Aug 2017 14:06:24 +0200 Subject: [PATCH] Added bit packing for serialization of vector --- include/storage/serialization.hpp | 47 ++++++++++++++++++---- unit_tests/util/serialization.cpp | 67 +++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 unit_tests/util/serialization.cpp diff --git a/include/storage/serialization.hpp b/include/storage/serialization.hpp index fbd7e4c5a..2afb835ac 100644 --- a/include/storage/serialization.hpp +++ b/include/storage/serialization.hpp @@ -114,44 +114,75 @@ template void write(io::FileWriter &writer, const util::vector_view writer.WriteFrom(data.data(), count); } +template +inline unsigned char packBits(const T &data, std::size_t index, std::size_t count) +{ + static_assert(std::is_same::value, "value_type is not bool"); + unsigned char value = 0; + for (std::size_t bit = 0; bit < count; ++bit, ++index) + value = (value << 1) | data[index]; + return value; +} + +template +inline void unpackBits(T &data, std::size_t index, std::size_t count, unsigned char value) +{ + static_assert(std::is_same::value, "value_type is not bool"); + const unsigned char mask = 1 << (count - 1); + for (std::size_t bit = 0; bit < count; value <<= 1, ++bit, ++index) + data[index] = value & mask; +} + template <> inline void read(io::FileReader &reader, util::vector_view &data) { const auto count = reader.ReadElementCount64(); BOOST_ASSERT(data.size() == count); - for (const auto index : util::irange(0, count)) + std::uint64_t index = 0; + for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT) { - data[index] = reader.ReadOne(); + unpackBits(data, index, CHAR_BIT, reader.ReadOne()); } + if (count > index) + unpackBits(data, index, count - index, reader.ReadOne()); } template <> inline void write(io::FileWriter &writer, const util::vector_view &data) { const auto count = data.size(); writer.WriteElementCount64(count); - for (const auto index : util::irange(0, count)) + std::uint64_t index = 0; + for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT) { - writer.WriteOne(data[index]); + writer.WriteOne(packBits(data, CHAR_BIT * index, CHAR_BIT)); } + if (count > index) + writer.WriteOne(packBits(data, index, count - index)); } template <> inline void read(io::FileReader &reader, std::vector &data) { const auto count = reader.ReadElementCount64(); data.resize(count); - for (const auto index : util::irange(0, count)) + std::uint64_t index = 0; + for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT) { - data[index] = reader.ReadOne(); + unpackBits(data, index, CHAR_BIT, reader.ReadOne()); } + if (count > index) + unpackBits(data, index, count - index, reader.ReadOne()); } template <> inline void write(io::FileWriter &writer, const std::vector &data) { const auto count = data.size(); writer.WriteElementCount64(count); - for (const auto index : util::irange(0, count)) + std::uint64_t index = 0; + for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT) { - writer.WriteOne(data[index]); + writer.WriteOne(packBits(data, index, CHAR_BIT)); } + if (count > index) + writer.WriteOne(packBits(data, index, count - index)); } } } diff --git a/unit_tests/util/serialization.cpp b/unit_tests/util/serialization.cpp new file mode 100644 index 000000000..0db1a92fe --- /dev/null +++ b/unit_tests/util/serialization.cpp @@ -0,0 +1,67 @@ +#include "storage/serialization.hpp" + +#include +#include +#include +#include + +#include +#include + +BOOST_AUTO_TEST_SUITE(serialization_test) + +using namespace osrm; +using namespace osrm::util; +using namespace osrm::storage::io; +using namespace osrm::storage::serialization; + +BOOST_AUTO_TEST_CASE(pack_test) +{ + std::vector v = {0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1}; + + BOOST_CHECK_EQUAL(osrm::storage::serialization::packBits(v, 0, 8), 0x2e); + BOOST_CHECK_EQUAL(osrm::storage::serialization::packBits(v, 5, 7), 0x65); + BOOST_CHECK_EQUAL(osrm::storage::serialization::packBits(v, 6, 8), 0x95); + BOOST_CHECK_EQUAL(osrm::storage::serialization::packBits(v, 11, 1), 0x01); +} + +BOOST_AUTO_TEST_CASE(unpack_test) +{ + std::vector v(14), expected = {0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1}; + + osrm::storage::serialization::unpackBits(v, 0, 8, 0x2e); + osrm::storage::serialization::unpackBits(v, 5, 7, 0x65); + osrm::storage::serialization::unpackBits(v, 6, 8, 0x95); + osrm::storage::serialization::unpackBits(v, 11, 1, 0x01); + BOOST_CHECK_EQUAL_COLLECTIONS(v.begin(), v.end(), expected.begin(), expected.end()); +} + +struct SerializationFixture +{ + SerializationFixture() : temporary_file(boost::filesystem::unique_path()) {} + ~SerializationFixture() { remove(temporary_file); } + + boost::filesystem::path temporary_file; +}; + +BOOST_AUTO_TEST_CASE(serialize_bool_vector) +{ + SerializationFixture fixture; + { + std::vector> data = { + {}, {0}, {1, 1, 1}, {1, 1, 0, 0, 1, 1, 0, 0}, {1, 1, 0, 0, 1, 1, 0, 0, 1}}; + for (const auto &v : data) + { + { + FileWriter writer(fixture.temporary_file, FileWriter::GenerateFingerprint); + write(writer, v); + } + std::vector result; + FileReader reader(fixture.temporary_file, FileReader::VerifyFingerprint); + read(reader, result); + BOOST_CHECK_EQUAL_COLLECTIONS(v.begin(), v.end(), result.begin(), result.end()); + } + } +} + +BOOST_AUTO_TEST_SUITE_END()