#ifndef OSRM_STORAGE_SERIALIZATION_HPP #define OSRM_STORAGE_SERIALIZATION_HPP #include "util/deallocating_vector.hpp" #include "util/integer_range.hpp" #include "util/vector_view.hpp" #include "storage/io.hpp" #include #include #if USE_STXXL_LIBRARY #include #endif namespace osrm { namespace storage { namespace serialization { /* All vector formats here use the same on-disk format. * This is important because we want to be able to write from a vector * of one kind, but read it into a vector of another kind. * * All vector types with this guarantee should be placed in this file. */ template inline void read(storage::io::FileReader &reader, util::DeallocatingVector &vec) { vec.current_size = reader.ReadElementCount64(vec.current_size); std::size_t num_blocks = std::ceil(vec.current_size / util::DeallocatingVector::ELEMENTS_PER_BLOCK); vec.bucket_list.resize(num_blocks); // Read all but the last block which can be partital for (auto bucket_index : util::irange(0, num_blocks - 1)) { vec.bucket_list[bucket_index] = new T[util::DeallocatingVector::ELEMENTS_PER_BLOCK]; reader.ReadInto(vec.bucket_list[bucket_index], util::DeallocatingVector::ELEMENTS_PER_BLOCK); } std::size_t last_block_size = vec.current_size % util::DeallocatingVector::ELEMENTS_PER_BLOCK; vec.bucket_list.back() = new T[util::DeallocatingVector::ELEMENTS_PER_BLOCK]; reader.ReadInto(vec.bucket_list.back(), last_block_size); } template inline void write(storage::io::FileWriter &writer, const util::DeallocatingVector &vec) { writer.WriteElementCount64(vec.current_size); // Write all but the last block which can be partially filled for (auto bucket_index : util::irange(0, vec.bucket_list.size() - 1)) { writer.WriteFrom(vec.bucket_list[bucket_index], util::DeallocatingVector::ELEMENTS_PER_BLOCK); } std::size_t last_block_size = vec.current_size % util::DeallocatingVector::ELEMENTS_PER_BLOCK; writer.WriteFrom(vec.bucket_list.back(), last_block_size); } #if USE_STXXL_LIBRARY template inline void read(storage::io::FileReader &reader, stxxl::vector &vec) { auto size = reader.ReadOne(); vec.reserve(size); for (auto idx : util::irange(0, size)) { (void)idx; vec.push_back(reader.ReadOne()); } } template inline void write(storage::io::FileWriter &writer, const stxxl::vector &vec) { writer.WriteOne(vec.size()); for (auto idx : util::irange(0, vec.size())) { writer.WriteOne(vec[idx]); } } #endif template void read(io::FileReader &reader, std::vector &data) { const auto count = reader.ReadElementCount64(); data.resize(count); reader.ReadInto(data.data(), count); } template void write(io::FileWriter &writer, const std::vector &data) { const auto count = data.size(); writer.WriteElementCount64(count); writer.WriteFrom(data.data(), count); } template void read(io::FileReader &reader, util::vector_view &data) { const auto count = reader.ReadElementCount64(); BOOST_ASSERT(data.size() == count); reader.ReadInto(data.data(), count); } template void write(io::FileWriter &writer, const util::vector_view &data) { const auto count = data.size(); writer.WriteElementCount64(count); writer.WriteFrom(data.data(), count); } template inline unsigned char packBits(const T &data, std::size_t index, std::size_t count) { static_assert(std::is_same::value, "value_type is not bool"); unsigned char value = 0; for (std::size_t bit = 0; bit < count; ++bit, ++index) value = (value << 1) | data[index]; return value; } template inline void unpackBits(T &data, std::size_t index, std::size_t count, unsigned char value) { static_assert(std::is_same::value, "value_type is not bool"); const unsigned char mask = 1 << (count - 1); for (std::size_t bit = 0; bit < count; value <<= 1, ++bit, ++index) data[index] = value & mask; } template <> inline void read(io::FileReader &reader, util::vector_view &data) { const auto count = reader.ReadElementCount64(); BOOST_ASSERT(data.size() == count); std::uint64_t index = 0; for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT) { unpackBits(data, index, CHAR_BIT, reader.ReadOne()); } if (count > index) unpackBits(data, index, count - index, reader.ReadOne()); } template <> inline void write(io::FileWriter &writer, const util::vector_view &data) { const auto count = data.size(); writer.WriteElementCount64(count); std::uint64_t index = 0; for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT) { writer.WriteOne(packBits(data, CHAR_BIT * index, CHAR_BIT)); } if (count > index) writer.WriteOne(packBits(data, index, count - index)); } template <> inline void read(io::FileReader &reader, std::vector &data) { const auto count = reader.ReadElementCount64(); data.resize(count); std::uint64_t index = 0; for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT) { unpackBits(data, index, CHAR_BIT, reader.ReadOne()); } if (count > index) unpackBits(data, index, count - index, reader.ReadOne()); } template <> inline void write(io::FileWriter &writer, const std::vector &data) { const auto count = data.size(); writer.WriteElementCount64(count); std::uint64_t index = 0; for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT) { writer.WriteOne(packBits(data, index, CHAR_BIT)); } if (count > index) writer.WriteOne(packBits(data, index, count - index)); } } } } #endif