Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Remove old io::FileWrite serialization code

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-22 17:34:26 +00:00
parent f1a392c4df
commit a542da3678
9 changed files with 20 additions and 425 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
include/extractor/serialization.hpp
View File

@ -132,143 +132,6 @@ inline void write(storage::tar::FileWriter &writer,
writer, name + "/annotations", node_data_container.annotation_data);
}
inline void read(storage::io::FileReader &reader, NodeRestriction &restriction)
{
reader.ReadInto(restriction.from);
reader.ReadInto(restriction.via);
reader.ReadInto(restriction.to);
}
inline void write(storage::io::FileWriter &writer, const NodeRestriction &restriction)
{
writer.WriteFrom(restriction.from);
writer.WriteFrom(restriction.via);
writer.WriteFrom(restriction.to);
}
inline void read(storage::io::FileReader &reader, WayRestriction &restriction)
{
read(reader, restriction.in_restriction);
read(reader, restriction.out_restriction);
}
inline void write(storage::io::FileWriter &writer, const WayRestriction &restriction)
{
write(writer, restriction.in_restriction);
write(writer, restriction.out_restriction);
}
inline void read(storage::io::FileReader &reader, TurnRestriction &restriction)
{
reader.ReadInto(restriction.is_only);
std::uint32_t restriction_type;
reader.ReadInto(restriction_type);
if (restriction_type == RestrictionType::WAY_RESTRICTION)
{
WayRestriction way_restriction;
read(reader, way_restriction);
restriction.node_or_way = std::move(way_restriction);
}
else
{
BOOST_ASSERT(restriction_type == RestrictionType::NODE_RESTRICTION);
NodeRestriction node_restriction;
read(reader, node_restriction);
restriction.node_or_way = std::move(node_restriction);
}
}
inline void write(storage::io::FileWriter &writer, const TurnRestriction &restriction)
{
writer.WriteFrom(restriction.is_only);
const std::uint32_t restriction_type = restriction.Type();
writer.WriteFrom(restriction_type);
if (restriction.Type() == RestrictionType::WAY_RESTRICTION)
{
write(writer, mapbox::util::get<WayRestriction>(restriction.node_or_way));
}
else
{
BOOST_ASSERT(restriction.Type() == RestrictionType::NODE_RESTRICTION);
write(writer, mapbox::util::get<NodeRestriction>(restriction.node_or_way));
}
}
inline void write(storage::io::FileWriter &writer, const ConditionalTurnRestriction &restriction)
{
write(writer, static_cast<const TurnRestriction &>(restriction));
writer.WriteElementCount64(restriction.condition.size());
for (const auto &c : restriction.condition)
{
writer.WriteFrom(c.modifier);
storage::serialization::write(writer, c.times);
storage::serialization::write(writer, c.weekdays);
storage::serialization::write(writer, c.monthdays);
}
}
inline void read(storage::io::FileReader &reader, ConditionalTurnRestriction &restriction)
{
read(reader, static_cast<TurnRestriction &>(restriction));
const auto num_conditions = reader.ReadElementCount64();
restriction.condition.resize(num_conditions);
for (uint64_t i = 0; i < num_conditions; i++)
{
reader.ReadInto(restriction.condition[i].modifier);
storage::serialization::read(reader, restriction.condition[i].times);
storage::serialization::read(reader, restriction.condition[i].weekdays);
storage::serialization::read(reader, restriction.condition[i].monthdays);
}
}
// read/write for conditional turn restrictions file
inline void read(storage::io::FileReader &reader, std::vector<TurnRestriction> &restrictions)
{
auto num_indices = reader.ReadElementCount64();
restrictions.reserve(num_indices);
TurnRestriction restriction;
while (num_indices-- > 0)
{
read(reader, restriction);
restrictions.push_back(std::move(restriction));
}
}
inline void write(storage::io::FileWriter &writer, const std::vector<TurnRestriction> &restrictions)
{
const auto num_indices = restrictions.size();
writer.WriteElementCount64(num_indices);
const auto write_restriction = [&writer](const auto &restriction) {
write(writer, restriction);
};
std::for_each(restrictions.begin(), restrictions.end(), write_restriction);
}
// read/write for conditional turn restrictions file
inline void read(storage::io::FileReader &reader,
std::vector<ConditionalTurnRestriction> &restrictions)
{
auto num_indices = reader.ReadElementCount64();
restrictions.reserve(num_indices);
ConditionalTurnRestriction restriction;
while (num_indices-- > 0)
{
read(reader, restriction);
restrictions.push_back(std::move(restriction));
}
}
inline void write(storage::io::FileWriter &writer,
const std::vector<ConditionalTurnRestriction> &restrictions)
{
const auto num_indices = restrictions.size();
writer.WriteElementCount64(num_indices);
const auto write_restriction = [&writer](const auto &restriction) {
write(writer, restriction);
};
std::for_each(restrictions.begin(), restrictions.end(), write_restriction);
}
inline void read(storage::io::BufferReader &reader, ConditionalTurnPenalty &turn_penalty)
{
reader.ReadInto(turn_penalty.turn_offset);

135
include/storage/serialization.hpp
View File

@ -32,41 +32,6 @@ namespace serialization
* All vector types with this guarantee should be placed in this file.
*/
template <typename T>
inline void read(storage::io::FileReader &reader, util::DeallocatingVector<T> &vec)
{
vec.current_size = reader.ReadElementCount64(vec.current_size);
std::size_t num_blocks =
std::ceil(vec.current_size / util::DeallocatingVector<T>::ELEMENTS_PER_BLOCK);
vec.bucket_list.resize(num_blocks);
// Read all but the last block which can be partital
for (auto bucket_index : util::irange<std::size_t>(0, num_blocks - 1))
{
vec.bucket_list[bucket_index] = new T[util::DeallocatingVector<T>::ELEMENTS_PER_BLOCK];
reader.ReadInto(vec.bucket_list[bucket_index],
util::DeallocatingVector<T>::ELEMENTS_PER_BLOCK);
}
std::size_t last_block_size =
vec.current_size % util::DeallocatingVector<T>::ELEMENTS_PER_BLOCK;
vec.bucket_list.back() = new T[util::DeallocatingVector<T>::ELEMENTS_PER_BLOCK];
reader.ReadInto(vec.bucket_list.back(), last_block_size);
}
template <typename T>
inline void write(storage::io::FileWriter &writer, const util::DeallocatingVector<T> &vec)
{
writer.WriteElementCount64(vec.current_size);
// Write all but the last block which can be partially filled
for (auto bucket_index : util::irange<std::size_t>(0, vec.bucket_list.size() - 1))
{
writer.WriteFrom(vec.bucket_list[bucket_index],
util::DeallocatingVector<T>::ELEMENTS_PER_BLOCK);
}
std::size_t last_block_size =
vec.current_size % util::DeallocatingVector<T>::ELEMENTS_PER_BLOCK;
writer.WriteFrom(vec.bucket_list.back(), last_block_size);
}
template <typename T>
inline void read(storage::tar::FileReader &reader, const std::string& name, util::DeallocatingVector<T> &vec)
{
@ -82,27 +47,18 @@ inline void write(storage::tar::FileWriter &writer, const std::string& name, con
}
#if USE_STXXL_LIBRARY
template <typename T> inline void read(storage::io::FileReader &reader, stxxl::vector<T> &vec)
template <typename T> inline void read(storage::tar::FileReader &reader, const std::string& name, stxxl::vector<T> &vec)
{
auto size = reader.ReadElementCount64();
auto size = reader.ReadElementCount64(name);
vec.reserve(size);
T tmp;
for (auto idx : util::irange<std::size_t>(0, size))
{
(void)idx;
reader.ReadInto(tmp);
vec.push_back(tmp);
}
reader.ReadStreaming<T>(name, std::back_inserter(vec), size);
}
template <typename T>
inline void write(storage::io::FileWriter &writer, const stxxl::vector<T> &vec)
inline void write(storage::tar::FileWriter &writer, const std::string& name, const stxxl::vector<T> &vec)
{
writer.WriteFrom(vec.size());
for (auto idx : util::irange<std::size_t>(0, vec.size()))
{
writer.WriteFrom<T>(vec[idx]);
}
writer.WriteElementCount64(name, vec.size());
writer.WriteStreaming<T>(name, vec.begin(), vec.size());
}
#endif
@ -168,34 +124,6 @@ void write(tar::FileWriter &writer, const std::string &name, const util::vector_
writer.WriteFrom(name, data.data(), count);
}
template <typename T> void read(io::FileReader &reader, std::vector<T> &data)
{
const auto count = reader.ReadElementCount64();
data.resize(count);
reader.ReadInto(data.data(), count);
}
template <typename T> void write(io::FileWriter &writer, const std::vector<T> &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
writer.WriteFrom(data.data(), count);
}
template <typename T> void read(io::FileReader &reader, util::vector_view<T> &data)
{
const auto count = reader.ReadElementCount64();
data.resize(count);
reader.ReadInto(data.data(), count);
}
template <typename T> void write(io::FileWriter &writer, const util::vector_view<T> &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
writer.WriteFrom(data.data(), count);
}
namespace detail
{
template <typename T>
@ -217,37 +145,6 @@ inline void unpackBits(T &data, std::size_t index, std::size_t count, unsigned c
data[index] = value & mask;
}
template <typename VectorT> void readBoolVector(io::FileReader &reader, VectorT &data)
{
const auto count = reader.ReadElementCount64();
data.resize(count);
std::uint64_t index = 0;
unsigned char block_data;
for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
{
reader.ReadInto(block_data);
unpackBits(data, index, CHAR_BIT, block_data);
}
if (count > index)
{
reader.ReadInto(block_data);
unpackBits(data, index, count - index, block_data);
}
}
template <typename VectorT> void writeBoolVector(io::FileWriter &writer, const VectorT &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
std::uint64_t index = 0;
for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
{
writer.WriteFrom<unsigned char>(packBits(data, index, CHAR_BIT));
}
if (count > index)
writer.WriteFrom<unsigned char>(packBits(data, index, count - index));
}
template <typename VectorT>
void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data)
{
@ -286,26 +183,6 @@ void writeBoolVector(tar::FileWriter &writer, const std::string &name, const Vec
}
}
template <> inline void read<bool>(io::FileReader &reader, util::vector_view<bool> &data)
{
detail::readBoolVector(reader, data);
}
template <> inline void write<bool>(io::FileWriter &writer, const util::vector_view<bool> &data)
{
detail::writeBoolVector(writer, data);
}
template <> inline void read<bool>(io::FileReader &reader, std::vector<bool> &data)
{
detail::readBoolVector(reader, data);
}
template <> inline void write<bool>(io::FileWriter &writer, const std::vector<bool> &data)
{
detail::writeBoolVector(writer, data);
}
template <>
inline void
read<bool>(tar::FileReader &reader, const std::string &name, util::vector_view<bool> &data)

24
include/util/deallocating_vector.hpp
View File

@ -13,23 +13,6 @@
namespace osrm
{
namespace util
{
template <typename ElementT> class DeallocatingVector;
}
namespace storage
{
namespace serialization
{
template <typename T>
inline void read(storage::io::FileReader &reader, util::DeallocatingVector<T> &vec);
template <typename T>
inline void write(storage::io::FileWriter &writer, const util::DeallocatingVector<T> &vec);
}
}
namespace util
{
template <typename ElementT> struct ConstDeallocatingVectorIteratorState
@ -183,6 +166,8 @@ class DeallocatingVectorIterator
}
};
template <typename ElementT> class DeallocatingVector;
template <typename T> void swap(DeallocatingVector<T> &lhs, DeallocatingVector<T> &rhs);
template <typename ElementT> class DeallocatingVector
@ -359,11 +344,6 @@ template <typename ElementT> class DeallocatingVector
++position;
}
}
friend void storage::serialization::read<ElementT>(storage::io::FileReader &reader,
DeallocatingVector &vec);
friend void storage::serialization::write<ElementT>(storage::io::FileWriter &writer,
const DeallocatingVector &vec);
};
template <typename T> void swap(DeallocatingVector<T> &lhs, DeallocatingVector<T> &rhs)

13
include/util/dynamic_graph.hpp
View File

@ -6,8 +6,6 @@
#include "util/permutation.hpp"
#include "util/typedefs.hpp"
#include "storage/io_fwd.hpp"
#include <boost/assert.hpp>
#include <cstdint>
@ -22,17 +20,6 @@ namespace osrm
{
namespace util
{
template <typename EdgeDataT> class DynamicGraph;
namespace serialization
{
template <typename EdgeDataT, bool UseSharedMemory>
void read(storage::io::FileReader &reader, DynamicGraph<EdgeDataT> &graph);
template <typename EdgeDataT, bool UseSharedMemory>
void write(storage::io::FileWriter &writer, const DynamicGraph<EdgeDataT> &graph);
}
namespace detail
{
// These types need to live outside of DynamicGraph

14
include/util/packed_vector.hpp
View File

@ -5,7 +5,6 @@
#include "util/typedefs.hpp"
#include "util/vector_view.hpp"
#include "storage/io_fwd.hpp"
#include "storage/shared_memory_ownership.hpp"
#include "storage/tar_fwd.hpp"
@ -29,13 +28,6 @@ template <typename T, std::size_t Bits, storage::Ownership Ownership> class Pack
namespace serialization
{
template <typename T, std::size_t Bits, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Bits, Ownership> &vec);
template <typename T, std::size_t Bits, storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::PackedVector<T, Bits, Ownership> &vec);
template <typename T, std::size_t Bits, storage::Ownership Ownership>
inline void read(storage::tar::FileReader &reader,
const std::string &name,
@ -457,12 +449,6 @@ template <typename T, std::size_t Bits, storage::Ownership Ownership> class Pack
vec.reserve(num_blocks * BLOCK_WORDS + 1);
}
friend void serialization::read<T, Bits, Ownership>(storage::io::FileReader &reader,
PackedVector &vec);
friend void serialization::write<T, Bits, Ownership>(storage::io::FileWriter &writer,
const PackedVector &vec);
friend void serialization::read<T, Bits, Ownership>(storage::tar::FileReader &reader,
const std::string &name,
PackedVector &vec);

71
include/util/serialization.hpp
View File

@ -17,21 +17,6 @@ namespace util
{
namespace serialization
{
template <unsigned BlockSize, storage::Ownership Ownership>
void write(storage::io::FileWriter &writer, const util::RangeTable<BlockSize, Ownership> &table)
{
writer.WriteFrom(table.sum_lengths);
storage::serialization::write(writer, table.block_offsets);
storage::serialization::write(writer, table.diff_blocks);
}
template <unsigned BlockSize, storage::Ownership Ownership>
void read(storage::io::FileReader &reader, util::RangeTable<BlockSize, Ownership> &table)
{
reader.ReadInto(table.sum_lengths);
storage::serialization::read(reader, table.block_offsets);
storage::serialization::read(reader, table.diff_blocks);
}
template <unsigned BlockSize, storage::Ownership Ownership>
void write(storage::tar::FileWriter &writer,
@ -53,21 +38,6 @@ void read(storage::tar::FileReader &reader,
storage::serialization::read(reader, name + "/diff_blocks", table.diff_blocks);
}
template <typename T, std::size_t Bits, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Bits, Ownership> &vec)
{
reader.ReadInto(vec.num_elements);
storage::serialization::read(reader, vec.vec);
}
template <typename T, std::size_t Bits, storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::PackedVector<T, Bits, Ownership> &vec)
{
writer.WriteFrom(vec.num_elements);
storage::serialization::write(writer, vec.vec);
}
template <typename T, std::size_t Bits, storage::Ownership Ownership>
inline void read(storage::tar::FileReader &reader,
const std::string &name,
@ -86,47 +56,6 @@ inline void write(storage::tar::FileWriter &writer,
storage::serialization::write(writer, name + "/packed", vec.vec);
}
template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, StaticGraph<EdgeDataT, Ownership> &graph)
{
storage::serialization::read(reader, graph.node_array);
storage::serialization::read(reader, graph.edge_array);
graph.number_of_nodes = graph.node_array.size() - 1;
graph.number_of_edges = graph.edge_array.size();
}
template <typename EdgeDataT, storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, const StaticGraph<EdgeDataT, Ownership> &graph)
{
storage::serialization::write(writer, graph.node_array);
storage::serialization::write(writer, graph.edge_array);
}
template <typename EdgeDataT>
inline void read(storage::io::FileReader &reader, DynamicGraph<EdgeDataT> &graph)
{
storage::serialization::read(reader, graph.node_array);
const auto num_edges = reader.ReadElementCount64();
graph.edge_list.resize(num_edges);
for (auto index : irange<std::size_t>(0, num_edges))
{
reader.ReadInto(graph.edge_list[index]);
}
graph.number_of_nodes = graph.node_array.size();
graph.number_of_edges = num_edges;
}
template <typename EdgeDataT>
inline void write(storage::io::FileWriter &writer, const DynamicGraph<EdgeDataT> &graph)
{
storage::serialization::write(writer, graph.node_array);
writer.WriteElementCount64(graph.number_of_edges);
for (auto index : irange<std::size_t>(0, graph.number_of_edges))
{
writer.WriteFrom(graph.edge_list[index]);
}
}
template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::tar::FileReader &reader,
const std::string &name,

13
include/util/static_graph.hpp
View File

@ -8,7 +8,6 @@
#include "util/typedefs.hpp"
#include "util/vector_view.hpp"
#include "storage/io_fwd.hpp"
#include "storage/shared_memory_ownership.hpp"
#include "storage/tar_fwd.hpp"
@ -28,12 +27,6 @@ template <typename EdgeDataT, storage::Ownership Ownership> class StaticGraph;
namespace serialization
{
template <typename EdgeDataT, storage::Ownership Ownership>
void read(storage::io::FileReader &reader, StaticGraph<EdgeDataT, Ownership> &graph);
template <typename EdgeDataT, storage::Ownership Ownership>
void write(storage::io::FileWriter &writer, const StaticGraph<EdgeDataT, Ownership> &graph);
template <typename EdgeDataT, storage::Ownership Ownership>
void read(storage::tar::FileReader &reader,
const std::string &name,
@ -283,12 +276,6 @@ class StaticGraph
util::inplacePermutation(edge_array.begin(), edge_array.end(), old_to_new_edge);
}
friend void serialization::read<EdgeDataT, Ownership>(storage::io::FileReader &reader,
StaticGraph<EdgeDataT, Ownership> &graph);
friend void
serialization::write<EdgeDataT, Ownership>(storage::io::FileWriter &writer,
const StaticGraph<EdgeDataT, Ownership> &graph);
friend void serialization::read<EdgeDataT, Ownership>(storage::tar::FileReader &reader,
const std::string &name,
StaticGraph<EdgeDataT, Ownership> &graph);

20
unit_tests/storage/serialization.cpp
View File

@ -32,26 +32,6 @@ BOOST_AUTO_TEST_CASE(unpack_test)
BOOST_CHECK_EQUAL_COLLECTIONS(v.begin(), v.end(), expected.begin(), expected.end());
}
BOOST_AUTO_TEST_CASE(bin_serialize_bool_vector)
{
TemporaryFile tmp;
{
std::vector<std::vector<bool>> data = {
{}, {0}, {1, 1, 1}, {1, 1, 0, 0, 1, 1, 0, 0}, {1, 1, 0, 0, 1, 1, 0, 0, 1}};
for (const auto &v : data)
{
{
io::FileWriter writer(tmp.path, io::FileWriter::GenerateFingerprint);
storage::serialization::write(writer, v);
}
std::vector<bool> result;
io::FileReader reader(tmp.path, io::FileReader::VerifyFingerprint);
storage::serialization::read(reader, result);
BOOST_CHECK_EQUAL_COLLECTIONS(v.begin(), v.end(), result.begin(), result.end());
}
}
}
BOOST_AUTO_TEST_CASE(tar_serialize_bool_vector)
{
TemporaryFile tmp;

18
unit_tests/util/io.cpp
View File

@ -32,12 +32,14 @@ BOOST_AUTO_TEST_CASE(io_data)
{
osrm::storage::io::FileWriter outfile(IO_TMP_FILE,
osrm::storage::io::FileWriter::GenerateFingerprint);
osrm::storage::serialization::write(outfile, data_in);
outfile.WriteElementCount64(data_in.size());
outfile.WriteFrom(data_in.data(), data_in.size());
}
osrm::storage::io::FileReader infile(IO_TMP_FILE,
osrm::storage::io::FileReader::VerifyFingerprint);
osrm::storage::serialization::read(infile, data_out);
data_out.resize(infile.ReadElementCount64());
infile.ReadInto(data_out.data(), data_out.size());
BOOST_REQUIRE_EQUAL(data_in.size(), data_out.size());
BOOST_CHECK_EQUAL_COLLECTIONS(data_out.begin(), data_out.end(), data_in.begin(), data_in.end());
@ -88,7 +90,8 @@ BOOST_AUTO_TEST_CASE(file_too_small)
{
osrm::storage::io::FileWriter outfile(
IO_TOO_SMALL_FILE, osrm::storage::io::FileWriter::GenerateFingerprint);
osrm::storage::serialization::write(outfile, v);
outfile.WriteElementCount64(v.size());
outfile.WriteFrom(v.data(), v.size());
}
std::fstream f(IO_TOO_SMALL_FILE);
@ -102,7 +105,8 @@ BOOST_AUTO_TEST_CASE(file_too_small)
osrm::storage::io::FileReader infile(IO_TOO_SMALL_FILE,
osrm::storage::io::FileReader::VerifyFingerprint);
std::vector<int> buffer;
osrm::storage::serialization::read(infile, buffer);
buffer.resize(infile.ReadElementCount64());
infile.ReadInto(buffer.data(), buffer.size());
BOOST_REQUIRE_MESSAGE(false, "Should not get here");
}
catch (const osrm::util::RuntimeError &e)
@ -124,7 +128,8 @@ BOOST_AUTO_TEST_CASE(io_corrupt_fingerprint)
osrm::storage::io::FileWriter::HasNoFingerprint);
outfile.WriteFrom(0xDEADBEEFCAFEFACE);
osrm::storage::serialization::write(outfile, v);
outfile.WriteElementCount64(v.size());
outfile.WriteFrom(v.data(), v.size());
}
try
@ -155,7 +160,8 @@ BOOST_AUTO_TEST_CASE(io_incompatible_fingerprint)
const auto fingerprint = osrm::util::FingerPrint::GetValid();
outfile.WriteFrom(fingerprint);
osrm::storage::serialization::write(outfile, v);
outfile.WriteElementCount64(v.size());
outfile.WriteFrom(v.data(), v.size());
}
std::fstream f(IO_INCOMPATIBLE_FINGERPRINT_FILE);