Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add BufferWriter/BufferReader and simplify interface for ConditionalRestrictions

Browse Source
This commit is contained in:
Patrick Niklaus
2018-03-19 23:31:49 +00:00
parent 4f454a3761
commit 06f28ffd34
18 changed files with 237 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/storage/io.hpp
+52 -22
View File
@@ -6,6 +6,7 @@
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
#include "util/fingerprint.hpp"
#include "util/integer_range.hpp"
#include "util/log.hpp"
#include "util/version.hpp"
@@ -110,6 +111,38 @@ class FileReader
}
}
template <typename T, typename OutIter> void ReadStreaming(OutIter out, const std::size_t count)
{
#if !defined(__GNUC__) || (__GNUC__ > 4)
static_assert(!std::is_pointer<T>::value, "saving pointer types is not allowed");
static_assert(std::is_trivially_copyable<T>::value,
"bytewise reading requires trivially copyable type");
#endif
if (count == 0)
return;
T tmp;
for (auto index : util::irange<std::size_t>(0, count))
{
(void)index;
const auto &result = input_stream.read(reinterpret_cast<char *>(&tmp), sizeof(T));
const std::size_t bytes_read = input_stream.gcount();
if (bytes_read != sizeof(T) && !result)
{
if (result.eof())
{
throw util::RuntimeError(
filepath.string(), ErrorCode::UnexpectedEndOfFile, SOURCE_REF);
}
throw util::RuntimeError(
filepath.string(), ErrorCode::FileReadError, SOURCE_REF, std::strerror(errno));
}
*out++ = tmp;
}
}
template <typename T> void ReadInto(std::vector<T> &target)
{
ReadInto(target.data(), target.size());
@@ -117,13 +150,6 @@ class FileReader
template <typename T> void ReadInto(T &target) { ReadInto(&target, 1); }
template <typename T> T ReadOne()
{
T tmp;
ReadInto(tmp);
return tmp;
}
template <typename T> void Skip(const std::size_t element_count)
{
boost::iostreams::seek(input_stream, element_count * sizeof(T), BOOST_IOS::cur);
@@ -131,7 +157,12 @@ class FileReader
/*******************************************/
std::uint64_t ReadElementCount64() { return ReadOne<std::uint64_t>(); }
std::uint64_t ReadElementCount64()
{
std::uint64_t count;
ReadInto(count);
return count;
}
template <typename T> std::size_t ReadVectorSize()
{
@@ -142,7 +173,8 @@ class FileReader
bool ReadAndCheckFingerprint()
{
auto loaded_fingerprint = ReadOne<util::FingerPrint>();
util::FingerPrint loaded_fingerprint;
ReadInto(loaded_fingerprint);
const auto expected_fingerprint = util::FingerPrint::GetValid();
if (!loaded_fingerprint.IsValid())
@@ -229,14 +261,12 @@ class FileWriter
template <typename T> void WriteFrom(const T &src) { WriteFrom(&src, 1); }
template <typename T> void WriteOne(const T &tmp) { WriteFrom(tmp); }
void WriteElementCount64(const std::uint64_t count) { WriteOne<std::uint64_t>(count); }
void WriteElementCount64(const std::uint64_t count) { WriteFrom(count); }
void WriteFingerprint()
{
const auto fingerprint = util::FingerPrint::GetValid();
return WriteOne(fingerprint);
return WriteFrom(fingerprint);
}
template <typename T> void Skip(const std::size_t element_count)
@@ -298,14 +328,14 @@ class BufferReader
}
}
template <typename T> T ReadOne()
{
T tmp;
ReadInto(&tmp, 1);
return tmp;
}
template <typename T> void ReadInto(T &tmp) { ReadInto(&tmp, 1); }
std::uint64_t ReadElementCount64() { return ReadOne<std::uint64_t>(); }
std::uint64_t ReadElementCount64()
{
std::uint64_t count;
ReadInto(count);
return count;
}
private:
std::istringstream input_stream;
@@ -343,9 +373,9 @@ class BufferWriter
}
}
template <typename T> void WriteOne(const T &tmp) { WriteFrom(&tmp, 1); }
template <typename T> void WriteFrom(const T &tmp) { WriteFrom(&tmp, 1); }
void WriteElementCount64(const std::uint64_t count) { WriteOne<std::uint64_t>(count); }
void WriteElementCount64(const std::uint64_t count) { WriteFrom(count); }
std::string GetBuffer() const { return output_stream.str(); }
include/storage/serialization.hpp
+31 -10
View File
@@ -84,22 +84,24 @@ inline void write(storage::tar::FileWriter &writer, const std::string& name, con
#if USE_STXXL_LIBRARY
template <typename T> inline void read(storage::io::FileReader &reader, stxxl::vector<T> &vec)
{
auto size = reader.ReadOne<std::uint64_t>();
auto size = reader.ReadElementCount64();
vec.reserve(size);
T tmp;
for (auto idx : util::irange<std::size_t>(0, size))
{
(void)idx;
vec.push_back(reader.ReadOne<T>());
reader.ReadInto(tmp);
vec.push_back(tmp);
}
}
template <typename T>
inline void write(storage::io::FileWriter &writer, const stxxl::vector<T> &vec)
{
writer.WriteOne(vec.size());
writer.WriteFrom(vec.size());
for (auto idx : util::irange<std::size_t>(0, vec.size()))
{
writer.WriteOne<T>(vec[idx]);
writer.WriteFrom<T>(vec[idx]);
}
}
#endif
@@ -120,8 +122,22 @@ void write(io::BufferWriter &writer, const std::vector<T> &data)
writer.WriteFrom(data.data(), count);
}
inline void write(tar::FileWriter &writer, const std::string &name, const std::string &data)
{
const auto count = data.size();
writer.WriteElementCount64(name, count);
writer.WriteFrom(name, data.data(), count);
}
inline void read(tar::FileReader &reader, const std::string &name, std::string &data)
{
const auto count = reader.ReadElementCount64(name);
data.resize(count);
reader.ReadInto(name, const_cast<char*>(data.data()), count);
}
template <typename T>
void read(tar::FileReader &reader, const std::string &name, std::vector<T> &data)
inline void read(tar::FileReader &reader, const std::string& name, std::vector<T> &data)
{
const auto count = reader.ReadElementCount64(name);
data.resize(count);
@@ -140,7 +156,7 @@ template <typename T>
void read(tar::FileReader &reader, const std::string &name, util::vector_view<T> &data)
{
const auto count = reader.ReadElementCount64(name);
BOOST_ASSERT(data.size() == count);
data.resize(count);
reader.ReadInto(name, data.data(), count);
}
@@ -206,12 +222,17 @@ template <typename VectorT> void readBoolVector(io::FileReader &reader, VectorT
const auto count = reader.ReadElementCount64();
data.resize(count);
std::uint64_t index = 0;
unsigned char block_data;
for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
{
unpackBits(data, index, CHAR_BIT, reader.ReadOne<unsigned char>());
reader.ReadInto(block_data);
unpackBits(data, index, CHAR_BIT, block_data);
}
if (count > index)
unpackBits(data, index, count - index, reader.ReadOne<unsigned char>());
{
reader.ReadInto(block_data);
unpackBits(data, index, count - index, block_data);
}
}
template <typename VectorT> void writeBoolVector(io::FileWriter &writer, const VectorT &data)
@@ -221,10 +242,10 @@ template <typename VectorT> void writeBoolVector(io::FileWriter &writer, const V
std::uint64_t index = 0;
for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
{
writer.WriteOne<unsigned char>(packBits(data, index, CHAR_BIT));
writer.WriteFrom<unsigned char>(packBits(data, index, CHAR_BIT));
}
if (count > index)
writer.WriteOne<unsigned char>(packBits(data, index, count - index));
writer.WriteFrom<unsigned char>(packBits(data, index, count - index));
}
template <typename VectorT>
include/storage/tar.hpp
+9 -8
View File
@@ -47,14 +47,14 @@ class FileReader
std::uint64_t ReadElementCount64(const std::string &name)
{
return ReadOne<std::uint64_t>(name + ".meta");
std::uint64_t size;
ReadInto(name + ".meta", size);
return size;
}
template <typename T> T ReadOne(const std::string &name)
template <typename T> void ReadInto(const std::string &name, T& tmp)
{
T tmp;
ReadInto(name, &tmp, 1);
return tmp;
}
template <typename T, typename OutIter> void ReadStreaming(const std::string &name, OutIter out)
@@ -127,7 +127,8 @@ class FileReader
private:
bool ReadAndCheckFingerprint()
{
auto loaded_fingerprint = ReadOne<util::FingerPrint>("osrm_fingerprint.meta");
util::FingerPrint loaded_fingerprint;
ReadInto("osrm_fingerprint.meta", loaded_fingerprint);
const auto expected_fingerprint = util::FingerPrint::GetValid();
if (!loaded_fingerprint.IsValid())
@@ -183,10 +184,10 @@ class FileWriter
void WriteElementCount64(const std::string &name, const std::uint64_t count)
{
WriteOne(name + ".meta", count);
WriteFrom(name + ".meta", count);
}
template <typename T> void WriteOne(const std::string &name, const T &data)
template <typename T> void WriteFrom(const std::string &name, const T &data)
{
WriteFrom(name, &data, 1);
}
@@ -236,7 +237,7 @@ class FileWriter
void WriteFingerprint()
{
const auto fingerprint = util::FingerPrint::GetValid();
WriteOne("osrm_fingerprint.meta", fingerprint);
WriteFrom("osrm_fingerprint.meta", fingerprint);
}
boost::filesystem::path path;