Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Refactor metric storage

Browse Source
This commit is contained in:
Patrick Niklaus
2018-03-27 09:44:13 +00:00
committed by Patrick Niklaus
parent aec9b6a178
commit c334d11e95
14 changed files with 505 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/storage/serialization.hpp
+66 -65
View File
@@ -27,6 +27,72 @@ namespace storage
namespace serialization
{
namespace detail
{
template <typename T, typename BlockT = unsigned char>
inline BlockT packBits(const T &data, std::size_t index, std::size_t count)
{
static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
BlockT value = 0;
for (std::size_t bit = 0; bit < count; ++bit, ++index)
value = (value << 1) | data[index];
return value;
}
template <typename T, typename BlockT = unsigned char>
inline void unpackBits(T &data, std::size_t index, std::size_t count, BlockT value)
{
static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
const BlockT mask = BlockT{1} << (count - 1);
for (std::size_t bit = 0; bit < count; value <<= 1, ++bit, ++index)
data[index] = value & mask;
}
template <typename VectorT>
void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data)
{
const auto count = reader.ReadElementCount64(name);
data.resize(count);
std::uint64_t index = 0;
constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
const auto decode = [&](const std::uint64_t block) {
auto read_size = std::min<std::size_t>(count - index, WORD_BITS);
unpackBits<VectorT, std::uint64_t>(data, index, read_size, block);
index += WORD_BITS;
};
reader.ReadStreaming<std::uint64_t>(name, boost::make_function_output_iterator(decode));
}
template <typename VectorT>
void writeBoolVector(tar::FileWriter &writer, const std::string &name, const VectorT &data)
{
const auto count = data.size();
writer.WriteElementCount64(name, count);
std::uint64_t index = 0;
constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
// FIXME on old boost version the function_input_iterator does not work with lambdas
// so we need to wrap it in a function here.
const std::function<std::uint64_t()> encode_function = [&]() -> std::uint64_t {
auto write_size = std::min<std::size_t>(count - index, WORD_BITS);
auto packed = packBits<VectorT, std::uint64_t>(data, index, write_size);
index += WORD_BITS;
return packed;
};
std::uint64_t number_of_blocks = (count + WORD_BITS - 1) / WORD_BITS;
writer.WriteStreaming<std::uint64_t>(
name,
boost::make_function_input_iterator(encode_function, boost::infinite()),
number_of_blocks);
}
}
/* All vector formats here use the same on-disk format.
* This is important because we want to be able to write from a vector
* of one kind, but read it into a vector of another kind.
@@ -155,71 +221,6 @@ void write(tar::FileWriter &writer, const std::string &name, const util::vector_
writer.WriteFrom(name, data.data(), count);
}
namespace detail
{
template <typename T, typename BlockT = unsigned char>
inline BlockT packBits(const T &data, std::size_t index, std::size_t count)
{
static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
BlockT value = 0;
for (std::size_t bit = 0; bit < count; ++bit, ++index)
value = (value << 1) | data[index];
return value;
}
template <typename T, typename BlockT = unsigned char>
inline void unpackBits(T &data, std::size_t index, std::size_t count, BlockT value)
{
static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
const BlockT mask = BlockT{1} << (count - 1);
for (std::size_t bit = 0; bit < count; value <<= 1, ++bit, ++index)
data[index] = value & mask;
}
template <typename VectorT>
void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data)
{
const auto count = reader.ReadElementCount64(name);
data.resize(count);
std::uint64_t index = 0;
constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
const auto decode = [&](const std::uint64_t block) {
auto read_size = std::min<std::size_t>(count - index, WORD_BITS);
unpackBits<VectorT, std::uint64_t>(data, index, read_size, block);
index += WORD_BITS;
};
reader.ReadStreaming<std::uint64_t>(name, boost::make_function_output_iterator(decode));
}
template <typename VectorT>
void writeBoolVector(tar::FileWriter &writer, const std::string &name, const VectorT &data)
{
const auto count = data.size();
writer.WriteElementCount64(name, count);
std::uint64_t index = 0;
constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
// FIXME on old boost version the function_input_iterator does not work with lambdas
// so we need to wrap it in a function here.
const std::function<std::uint64_t()> encode_function = [&]() -> std::uint64_t {
auto write_size = std::min<std::size_t>(count - index, WORD_BITS);
auto packed = packBits<VectorT, std::uint64_t>(data, index, write_size);
index += WORD_BITS;
return packed;
};
std::uint64_t number_of_blocks = (count + WORD_BITS - 1) / WORD_BITS;
writer.WriteStreaming<std::uint64_t>(
name,
boost::make_function_input_iterator(encode_function, boost::infinite()),
number_of_blocks);
}
}
template <>
inline void
read<bool>(tar::FileReader &reader, const std::string &name, util::vector_view<bool> &data)
include/storage/shared_datatype.hpp
-2
View File
@@ -94,8 +94,6 @@ class DataLayout
{
char *start_canary_ptr = ptr - sizeof(CANARY);
char *end_canary_ptr = ptr + GetBlockSize(name);
std::cout << name << ": " << (long)(start_canary_ptr - shared_memory) << " -> "
<< (long)(end_canary_ptr - shared_memory) << std::endl;
std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
}