Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add BufferWriter/BufferReader and simplify interface for ConditionalRestrictions

Browse Source
This commit is contained in:
Patrick Niklaus 2018-03-19 23:31:49 +00:00
parent 4f454a3761
commit 06f28ffd34
18 changed files with 237 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/contractor/files.hpp
View File

@ -32,7 +32,7 @@ inline void readGraph(const boost::filesystem::path &path,
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
checksum = reader.ReadOne<std::uint32_t>("/ch/checksum");
reader.ReadInto("/ch/checksum", checksum);
util::serialization::read(reader, "/ch/contracted_graph", graph);
auto count = reader.ReadElementCount64("/ch/edge_filter");
@ -42,7 +42,7 @@ inline void readGraph(const boost::filesystem::path &path,
storage::serialization::read(reader, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
}
connectivity_checksum = reader.ReadOne<std::uint32_t>("/ch/connectivity_checksum");
reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum);
}
// writes .osrm.hsgr file
@ -63,7 +63,7 @@ inline void writeGraph(const boost::filesystem::path &path,
storage::tar::FileWriter writer{path, fingerprint};
writer.WriteElementCount64("/ch/checksum", 1);
writer.WriteOne("/ch/checksum", checksum);
writer.WriteFrom("/ch/checksum", checksum);
util::serialization::write(writer, "/ch/contracted_graph", graph);
writer.WriteElementCount64("/ch/edge_filter", edge_filter.size());
@ -73,7 +73,7 @@ inline void writeGraph(const boost::filesystem::path &path,
}
writer.WriteElementCount64("/ch/connectivity_checksum", 1);
writer.WriteOne("/ch/connectivity_checksum", connectivity_checksum);
writer.WriteFrom("/ch/connectivity_checksum", connectivity_checksum);
}
}
}

29
include/extractor/files.hpp
View File

@ -87,10 +87,10 @@ void writeEdgeBasedGraph(const boost::filesystem::path &path,
storage::tar::FileWriter writer(path, storage::tar::FileWriter::GenerateFingerprint);
writer.WriteElementCount64("/common/number_of_edge_based_nodes", 1);
writer.WriteOne("/common/number_of_edge_based_nodes", number_of_edge_based_nodes);
writer.WriteFrom("/common/number_of_edge_based_nodes", number_of_edge_based_nodes);
storage::serialization::write(writer, "/common/edge_based_edge_list", edge_based_edge_list);
writer.WriteElementCount64("/common/connectivity_checksum", 1);
writer.WriteOne("/common/connectivity_checksum", connectivity_checksum);
writer.WriteFrom("/common/connectivity_checksum", connectivity_checksum);
}
template <typename EdgeBasedEdgeVector>
@ -103,9 +103,9 @@ void readEdgeBasedGraph(const boost::filesystem::path &path,
storage::tar::FileReader reader(path, storage::tar::FileReader::VerifyFingerprint);
number_of_edge_based_nodes = reader.ReadOne<EdgeID>("/common/number_of_edge_based_nodes");
reader.ReadInto("/common/number_of_edge_based_nodes", number_of_edge_based_nodes);
storage::serialization::read(reader, "/common/edge_based_edge_list", edge_based_edge_list);
connectivity_checksum = reader.ReadOne<std::uint32_t>("/common/connectivity_checksum");
reader.ReadInto("/common/connectivity_checksum", connectivity_checksum);
}
// reads .osrm.nbg_nodes
@ -364,6 +364,27 @@ inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPen
storage::serialization::read(reader, "/common/turn_penalty/duration", turn_penalty);
}
// writes .osrm.restrictions
template <typename ConditionalRestrictionsT>
inline void writeConditionalRestrictions(const boost::filesystem::path &path,
const ConditionalRestrictionsT &conditional_restrictions)
{
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};
serialization::write(writer, "/common/conditional_restrictions", conditional_restrictions);
}
// read .osrm.restrictions
template <typename ConditionalRestrictionsT>
inline void readConditionalRestrictions(const boost::filesystem::path &path, ConditionalRestrictionsT &conditional_restrictions)
{
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};
serialization::read(reader, "/common/conditional_restrictions", conditional_restrictions);
}
}
}
}

65
include/extractor/serialization.hpp
View File

@ -52,7 +52,7 @@ inline void write(storage::tar::FileWriter &writer,
inline void
read(storage::tar::FileReader &reader, const std::string &name, ProfileProperties &properties)
{
properties = reader.ReadOne<ProfileProperties>(name);
reader.ReadInto(name, properties);
}
inline void write(storage::tar::FileWriter &writer,
@ -60,19 +60,19 @@ inline void write(storage::tar::FileWriter &writer,
const ProfileProperties &properties)
{
writer.WriteElementCount64(name, 1);
writer.WriteOne(name, properties);
writer.WriteFrom(name, properties);
}
// read/write for datasources file
inline void read(storage::tar::FileReader &reader, const std::string &name, Datasources &sources)
{
sources = reader.ReadOne<Datasources>(name);
reader.ReadInto(name, sources);
}
inline void write(storage::tar::FileWriter &writer, const std::string &name, Datasources &sources)
{
writer.WriteElementCount64(name, 1);
writer.WriteOne(name, sources);
writer.WriteFrom(name, sources);
}
// read/write for segment data file
@ -138,9 +138,9 @@ inline void read(storage::io::FileReader &reader, NodeRestriction &restriction)
inline void write(storage::io::FileWriter &writer, const NodeRestriction &restriction)
{
writer.WriteOne(restriction.from);
writer.WriteOne(restriction.via);
writer.WriteOne(restriction.to);
writer.WriteFrom(restriction.from);
writer.WriteFrom(restriction.via);
writer.WriteFrom(restriction.to);
}
inline void read(storage::io::FileReader &reader, WayRestriction &restriction)
@ -177,9 +177,9 @@ inline void read(storage::io::FileReader &reader, TurnRestriction &restriction)
inline void write(storage::io::FileWriter &writer, const TurnRestriction &restriction)
{
writer.WriteOne(restriction.is_only);
writer.WriteFrom(restriction.is_only);
const std::uint32_t restriction_type = restriction.Type();
writer.WriteOne(restriction_type);
writer.WriteFrom(restriction_type);
if (restriction.Type() == RestrictionType::WAY_RESTRICTION)
{
write(writer, mapbox::util::get<WayRestriction>(restriction.node_or_way));
@ -197,7 +197,7 @@ inline void write(storage::io::FileWriter &writer, const ConditionalTurnRestrict
writer.WriteElementCount64(restriction.condition.size());
for (const auto &c : restriction.condition)
{
writer.WriteOne(c.modifier);
writer.WriteFrom(c.modifier);
storage::serialization::write(writer, c.times);
storage::serialization::write(writer, c.weekdays);
storage::serialization::write(writer, c.monthdays);
@ -266,7 +266,7 @@ inline void write(storage::io::FileWriter &writer,
std::for_each(restrictions.begin(), restrictions.end(), write_restriction);
}
inline void read(storage::io::FileReader &reader, ConditionalTurnPenalty &turn_penalty)
inline void read(storage::io::BufferReader &reader, ConditionalTurnPenalty &turn_penalty)
{
reader.ReadInto(turn_penalty.turn_offset);
reader.ReadInto(turn_penalty.location.lat);
@ -282,36 +282,61 @@ inline void read(storage::io::FileReader &reader, ConditionalTurnPenalty &turn_p
}
}
inline void write(storage::io::FileWriter &writer, const ConditionalTurnPenalty &turn_penalty)
inline void write(storage::io::BufferWriter &writer, const ConditionalTurnPenalty &turn_penalty)
{
writer.WriteOne(turn_penalty.turn_offset);
writer.WriteOne(static_cast<util::FixedLatitude::value_type>(turn_penalty.location.lat));
writer.WriteOne(static_cast<util::FixedLongitude::value_type>(turn_penalty.location.lon));
writer.WriteFrom(turn_penalty.turn_offset);
writer.WriteFrom(static_cast<util::FixedLatitude::value_type>(turn_penalty.location.lat));
writer.WriteFrom(static_cast<util::FixedLongitude::value_type>(turn_penalty.location.lon));
writer.WriteElementCount64(turn_penalty.conditions.size());
for (const auto &c : turn_penalty.conditions)
{
writer.WriteOne(c.modifier);
writer.WriteFrom(c.modifier);
storage::serialization::write(writer, c.times);
storage::serialization::write(writer, c.weekdays);
storage::serialization::write(writer, c.monthdays);
}
}
inline void write(storage::io::FileWriter &writer,
inline void write(storage::io::BufferWriter &writer,
const std::vector<ConditionalTurnPenalty> &conditional_penalties)
{
writer.WriteElementCount64(conditional_penalties.size());
for (const auto &penalty : conditional_penalties)
{
write(writer, penalty);
}
}
inline void read(storage::io::FileReader &reader,
std::vector<ConditionalTurnPenalty> &conditional_penalties)
inline void read(storage::io::BufferReader &reader, std::vector<ConditionalTurnPenalty> &conditional_penalties)
{
auto const num_elements = reader.ReadElementCount64();
auto num_elements = reader.ReadElementCount64();
conditional_penalties.resize(num_elements);
for (auto &penalty : conditional_penalties)
{
read(reader, penalty);
}
}
inline void write(storage::tar::FileWriter &writer,
const std::string& name,
const std::vector<ConditionalTurnPenalty> &conditional_penalties)
{
storage::io::BufferWriter buffer_writer;
write(buffer_writer, conditional_penalties);
storage::serialization::write(writer, name, buffer_writer.GetBuffer());
}
inline void read(storage::tar::FileReader &reader,
const std::string& name,
std::vector<ConditionalTurnPenalty> &conditional_penalties)
{
std::string buffer;
storage::serialization::read(reader, name, buffer);
storage::io::BufferReader buffer_reader{buffer};
read(buffer_reader, conditional_penalties);
}
}

2
include/guidance/serialization.hpp
View File

@ -39,7 +39,7 @@ inline void write(storage::io::FileWriter &writer,
storage::serialization::write(writer, turn_data_container.entry_class_ids);
storage::serialization::write(writer, turn_data_container.pre_turn_bearings);
storage::serialization::write(writer, turn_data_container.post_turn_bearings);
writer.WriteOne(connectivity_checksum);
writer.WriteFrom(connectivity_checksum);
}
}
}

8
include/partitioner/serialization.hpp
View File

@ -28,7 +28,7 @@ inline void read(storage::tar::FileReader &reader,
storage::serialization::read(reader, name + "/node_array", graph.node_array);
storage::serialization::read(reader, name + "/edge_array", graph.edge_array);
storage::serialization::read(reader, name + "/node_to_edge_offset", graph.node_to_edge_offset);
connectivity_checksum = reader.ReadOne<std::uint32_t>(name + "/connectivity_checksum");
reader.ReadInto(name + "/connectivity_checksum", connectivity_checksum);
}
template <typename EdgeDataT, storage::Ownership Ownership>
@ -41,7 +41,7 @@ inline void write(storage::tar::FileWriter &writer,
storage::serialization::write(writer, name + "/edge_array", graph.edge_array);
storage::serialization::write(writer, name + "/node_to_edge_offset", graph.node_to_edge_offset);
writer.WriteElementCount64(name + "/connectivity_checksum", 1);
writer.WriteOne(name + "/connectivity_checksum", connectivity_checksum);
writer.WriteFrom(name + "/connectivity_checksum", connectivity_checksum);
}
template <storage::Ownership Ownership>
@ -49,7 +49,7 @@ inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
*mlp.level_data = reader.ReadOne<typename std::remove_reference_t<decltype(mlp)>::LevelData>(name + "/level_data");
reader.ReadInto(name + "/level_data", *mlp.level_data);
storage::serialization::read(reader, name + "/partition", mlp.partition);
storage::serialization::read(reader, name + "/cell_to_children", mlp.cell_to_children);
}
@ -60,7 +60,7 @@ inline void write(storage::tar::FileWriter &writer,
const detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
writer.WriteElementCount64(name + "/level_data", 1);
writer.WriteOne(name + "/level_data", *mlp.level_data);
writer.WriteFrom(name + "/level_data", *mlp.level_data);
storage::serialization::write(writer, name + "/partition", mlp.partition);
storage::serialization::write(writer, name + "/cell_to_children", mlp.cell_to_children);
}

74
include/storage/io.hpp
View File

@ -6,6 +6,7 @@
#include "util/exception.hpp"
#include "util/exception_utils.hpp"
#include "util/fingerprint.hpp"
#include "util/integer_range.hpp"
#include "util/log.hpp"
#include "util/version.hpp"
@ -110,6 +111,38 @@ class FileReader
}
}
template <typename T, typename OutIter> void ReadStreaming(OutIter out, const std::size_t count)
{
#if !defined(__GNUC__) || (__GNUC__ > 4)
static_assert(!std::is_pointer<T>::value, "saving pointer types is not allowed");
static_assert(std::is_trivially_copyable<T>::value,
"bytewise reading requires trivially copyable type");
#endif
if (count == 0)
return;
T tmp;
for (auto index : util::irange<std::size_t>(0, count))
{
(void)index;
const auto &result = input_stream.read(reinterpret_cast<char *>(&tmp), sizeof(T));
const std::size_t bytes_read = input_stream.gcount();
if (bytes_read != sizeof(T) && !result)
{
if (result.eof())
{
throw util::RuntimeError(
filepath.string(), ErrorCode::UnexpectedEndOfFile, SOURCE_REF);
}
throw util::RuntimeError(
filepath.string(), ErrorCode::FileReadError, SOURCE_REF, std::strerror(errno));
}
*out++ = tmp;
}
}
template <typename T> void ReadInto(std::vector<T> &target)
{
ReadInto(target.data(), target.size());
@ -117,13 +150,6 @@ class FileReader
template <typename T> void ReadInto(T &target) { ReadInto(&target, 1); }
template <typename T> T ReadOne()
{
T tmp;
ReadInto(tmp);
return tmp;
}
template <typename T> void Skip(const std::size_t element_count)
{
boost::iostreams::seek(input_stream, element_count * sizeof(T), BOOST_IOS::cur);
@ -131,7 +157,12 @@ class FileReader
/*******************************************/
std::uint64_t ReadElementCount64() { return ReadOne<std::uint64_t>(); }
std::uint64_t ReadElementCount64()
{
std::uint64_t count;
ReadInto(count);
return count;
}
template <typename T> std::size_t ReadVectorSize()
{
@ -142,7 +173,8 @@ class FileReader
bool ReadAndCheckFingerprint()
{
auto loaded_fingerprint = ReadOne<util::FingerPrint>();
util::FingerPrint loaded_fingerprint;
ReadInto(loaded_fingerprint);
const auto expected_fingerprint = util::FingerPrint::GetValid();
if (!loaded_fingerprint.IsValid())
@ -229,14 +261,12 @@ class FileWriter
template <typename T> void WriteFrom(const T &src) { WriteFrom(&src, 1); }
template <typename T> void WriteOne(const T &tmp) { WriteFrom(tmp); }
void WriteElementCount64(const std::uint64_t count) { WriteOne<std::uint64_t>(count); }
void WriteElementCount64(const std::uint64_t count) { WriteFrom(count); }
void WriteFingerprint()
{
const auto fingerprint = util::FingerPrint::GetValid();
return WriteOne(fingerprint);
return WriteFrom(fingerprint);
}
template <typename T> void Skip(const std::size_t element_count)
@ -298,14 +328,14 @@ class BufferReader
}
}
template <typename T> T ReadOne()
{
T tmp;
ReadInto(&tmp, 1);
return tmp;
}
template <typename T> void ReadInto(T &tmp) { ReadInto(&tmp, 1); }
std::uint64_t ReadElementCount64() { return ReadOne<std::uint64_t>(); }
std::uint64_t ReadElementCount64()
{
std::uint64_t count;
ReadInto(count);
return count;
}
private:
std::istringstream input_stream;
@ -343,9 +373,9 @@ class BufferWriter
}
}
template <typename T> void WriteOne(const T &tmp) { WriteFrom(&tmp, 1); }
template <typename T> void WriteFrom(const T &tmp) { WriteFrom(&tmp, 1); }
void WriteElementCount64(const std::uint64_t count) { WriteOne<std::uint64_t>(count); }
void WriteElementCount64(const std::uint64_t count) { WriteFrom(count); }
std::string GetBuffer() const { return output_stream.str(); }

41
include/storage/serialization.hpp
View File

@ -84,22 +84,24 @@ inline void write(storage::tar::FileWriter &writer, const std::string& name, con
#if USE_STXXL_LIBRARY
template <typename T> inline void read(storage::io::FileReader &reader, stxxl::vector<T> &vec)
{
auto size = reader.ReadOne<std::uint64_t>();
auto size = reader.ReadElementCount64();
vec.reserve(size);
T tmp;
for (auto idx : util::irange<std::size_t>(0, size))
{
(void)idx;
vec.push_back(reader.ReadOne<T>());
reader.ReadInto(tmp);
vec.push_back(tmp);
}
}
template <typename T>
inline void write(storage::io::FileWriter &writer, const stxxl::vector<T> &vec)
{
writer.WriteOne(vec.size());
writer.WriteFrom(vec.size());
for (auto idx : util::irange<std::size_t>(0, vec.size()))
{
writer.WriteOne<T>(vec[idx]);
writer.WriteFrom<T>(vec[idx]);
}
}
#endif
@ -120,8 +122,22 @@ void write(io::BufferWriter &writer, const std::vector<T> &data)
writer.WriteFrom(data.data(), count);
}
inline void write(tar::FileWriter &writer, const std::string &name, const std::string &data)
{
const auto count = data.size();
writer.WriteElementCount64(name, count);
writer.WriteFrom(name, data.data(), count);
}
inline void read(tar::FileReader &reader, const std::string &name, std::string &data)
{
const auto count = reader.ReadElementCount64(name);
data.resize(count);
reader.ReadInto(name, const_cast<char*>(data.data()), count);
}
template <typename T>
void read(tar::FileReader &reader, const std::string &name, std::vector<T> &data)
inline void read(tar::FileReader &reader, const std::string& name, std::vector<T> &data)
{
const auto count = reader.ReadElementCount64(name);
data.resize(count);
@ -140,7 +156,7 @@ template <typename T>
void read(tar::FileReader &reader, const std::string &name, util::vector_view<T> &data)
{
const auto count = reader.ReadElementCount64(name);
BOOST_ASSERT(data.size() == count);
data.resize(count);
reader.ReadInto(name, data.data(), count);
}
@ -206,12 +222,17 @@ template <typename VectorT> void readBoolVector(io::FileReader &reader, VectorT
const auto count = reader.ReadElementCount64();
data.resize(count);
std::uint64_t index = 0;
unsigned char block_data;
for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
{
unpackBits(data, index, CHAR_BIT, reader.ReadOne<unsigned char>());
reader.ReadInto(block_data);
unpackBits(data, index, CHAR_BIT, block_data);
}
if (count > index)
unpackBits(data, index, count - index, reader.ReadOne<unsigned char>());
{
reader.ReadInto(block_data);
unpackBits(data, index, count - index, block_data);
}
}
template <typename VectorT> void writeBoolVector(io::FileWriter &writer, const VectorT &data)
@ -221,10 +242,10 @@ template <typename VectorT> void writeBoolVector(io::FileWriter &writer, const V
std::uint64_t index = 0;
for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
{
writer.WriteOne<unsigned char>(packBits(data, index, CHAR_BIT));
writer.WriteFrom<unsigned char>(packBits(data, index, CHAR_BIT));
}
if (count > index)
writer.WriteOne<unsigned char>(packBits(data, index, count - index));
writer.WriteFrom<unsigned char>(packBits(data, index, count - index));
}
template <typename VectorT>

17
include/storage/tar.hpp
View File

@ -47,14 +47,14 @@ class FileReader
std::uint64_t ReadElementCount64(const std::string &name)
{
return ReadOne<std::uint64_t>(name + ".meta");
std::uint64_t size;
ReadInto(name + ".meta", size);
return size;
}
template <typename T> T ReadOne(const std::string &name)
template <typename T> void ReadInto(const std::string &name, T& tmp)
{
T tmp;
ReadInto(name, &tmp, 1);
return tmp;
}
template <typename T, typename OutIter> void ReadStreaming(const std::string &name, OutIter out)
@ -127,7 +127,8 @@ class FileReader
private:
bool ReadAndCheckFingerprint()
{
auto loaded_fingerprint = ReadOne<util::FingerPrint>("osrm_fingerprint.meta");
util::FingerPrint loaded_fingerprint;
ReadInto("osrm_fingerprint.meta", loaded_fingerprint);
const auto expected_fingerprint = util::FingerPrint::GetValid();
if (!loaded_fingerprint.IsValid())
@ -183,10 +184,10 @@ class FileWriter
void WriteElementCount64(const std::string &name, const std::uint64_t count)
{
WriteOne(name + ".meta", count);
WriteFrom(name + ".meta", count);
}
template <typename T> void WriteOne(const std::string &name, const T &data)
template <typename T> void WriteFrom(const std::string &name, const T &data)
{
WriteFrom(name, &data, 1);
}
@ -236,7 +237,7 @@ class FileWriter
void WriteFingerprint()
{
const auto fingerprint = util::FingerPrint::GetValid();
WriteOne("osrm_fingerprint.meta", fingerprint);
WriteFrom("osrm_fingerprint.meta", fingerprint);
}
boost::filesystem::path path;

28
include/util/graph_loader.hpp
View File

@ -15,6 +15,7 @@
#include <boost/assert.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/function_output_iterator.hpp>
#include <tbb/parallel_sort.h>
@ -49,27 +50,22 @@ NodeID loadNodesFromFile(storage::io::FileReader &file_reader,
coordinates.resize(number_of_nodes);
osm_node_ids.reserve(number_of_nodes);
extractor::QueryNode current_node;
for (NodeID i = 0; i < number_of_nodes; ++i)
{
file_reader.ReadInto(&current_node, 1);
coordinates[i].lon = current_node.lon;
coordinates[i].lat = current_node.lat;
auto index = 0;
auto decode = [&](const extractor::QueryNode &current_node) {
coordinates[index].lon = current_node.lon;
coordinates[index].lat = current_node.lat;
osm_node_ids.push_back(current_node.node_id);
}
index++;
};
file_reader.ReadStreaming<extractor::QueryNode>(boost::make_function_output_iterator(decode),
number_of_nodes);
auto num_barriers = file_reader.ReadElementCount64();
for (auto index = 0UL; index < num_barriers; ++index)
{
*barriers++ = file_reader.ReadOne<NodeID>();
}
file_reader.ReadStreaming<NodeID>(barriers, num_barriers);
auto num_lights = file_reader.ReadElementCount64();
for (auto index = 0UL; index < num_lights; ++index)
{
*traffic_signals++ = file_reader.ReadOne<NodeID>();
}
file_reader.ReadStreaming<NodeID>(traffic_signals, num_lights);
return number_of_nodes;
}

10
include/util/indexed_data.hpp
View File

@ -106,7 +106,7 @@ template <int N, typename T = std::string> struct VariableGroupBlock
prefix_length += byte_length;
}
out.WriteOne(refernce);
out.WriteFrom(refernce);
return prefix_length;
}
@ -187,7 +187,7 @@ template <int N, typename T = std::string> struct FixedGroupBlock
BOOST_ASSERT(data_offset <= std::numeric_limits<decltype(BlockReference::offset)>::max());
BlockReference refernce{static_cast<decltype(BlockReference::offset)>(data_offset)};
out.WriteOne(refernce);
out.WriteFrom(refernce);
return BLOCK_SIZE;
}
@ -268,7 +268,7 @@ template <typename GroupBlock> struct IndexedData
const BlocksNumberType number_of_blocks =
number_of_elements == 0 ? 0
: 1 + (std::distance(first, sentinel) - 1) / (BLOCK_SIZE + 1);
out.WriteOne(number_of_blocks);
out.WriteFrom(number_of_blocks);
// Write block references and compute the total data size that includes prefix and data
const GroupBlock block;
@ -282,7 +282,7 @@ template <typename GroupBlock> struct IndexedData
}
// Write the total data size
out.WriteOne(data_size);
out.WriteFrom(data_size);
// Write data blocks that are (prefix, data)
for (OffsetIterator curr = first, next = first; next != sentinel; curr = next)
@ -291,7 +291,7 @@ template <typename GroupBlock> struct IndexedData
block.WriteBlockPrefix(out, curr, next);
std::advance(next, std::min<diff_type>(1, std::distance(next, sentinel)));
std::for_each(
data + *curr, data + *next, [&out](const auto &element) { out.WriteOne(element); });
data + *curr, data + *next, [&out](const auto &element) { out.WriteFrom(element); });
}
}

20
include/util/serialization.hpp
View File

@ -18,7 +18,7 @@ namespace serialization
template <unsigned BlockSize, storage::Ownership Ownership>
void write(storage::io::FileWriter &writer, const util::RangeTable<BlockSize, Ownership> &table)
{
writer.WriteOne(table.sum_lengths);
writer.WriteFrom(table.sum_lengths);
storage::serialization::write(writer, table.block_offsets);
storage::serialization::write(writer, table.diff_blocks);
}
@ -26,7 +26,7 @@ void write(storage::io::FileWriter &writer, const util::RangeTable<BlockSize, Ow
template <unsigned BlockSize, storage::Ownership Ownership>
void read(storage::io::FileReader &reader, util::RangeTable<BlockSize, Ownership> &table)
{
table.sum_lengths = reader.ReadOne<unsigned>();
reader.ReadInto(table.sum_lengths);
storage::serialization::read(reader, table.block_offsets);
storage::serialization::read(reader, table.diff_blocks);
}
@ -36,7 +36,7 @@ void write(storage::tar::FileWriter &writer,
const std::string &name,
const util::RangeTable<BlockSize, Ownership> &table)
{
writer.WriteOne(name + "/sum_lengths.meta", table.sum_lengths);
writer.WriteFrom(name + "/sum_lengths.meta", table.sum_lengths);
storage::serialization::write(writer, name + "/block_offsets", table.block_offsets);
storage::serialization::write(writer, name + "/diff_blocks", table.diff_blocks);
}
@ -46,7 +46,7 @@ void read(storage::tar::FileReader &reader,
const std::string &name,
util::RangeTable<BlockSize, Ownership> &table)
{
table.sum_lengths = reader.ReadOne<unsigned>(name + "/sum_lengths.meta");
reader.ReadInto(name + "/sum_lengths.meta", table.sum_lengths);
storage::serialization::read(reader, name + "/block_offsets", table.block_offsets);
storage::serialization::read(reader, name + "/diff_blocks", table.diff_blocks);
}
@ -54,7 +54,7 @@ void read(storage::tar::FileReader &reader,
template <typename T, std::size_t Bits, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Bits, Ownership> &vec)
{
vec.num_elements = reader.ReadOne<std::uint64_t>();
reader.ReadInto(vec.num_elements);
storage::serialization::read(reader, vec.vec);
}
@ -62,7 +62,7 @@ template <typename T, std::size_t Bits, storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::PackedVector<T, Bits, Ownership> &vec)
{
writer.WriteOne(vec.num_elements);
writer.WriteFrom(vec.num_elements);
storage::serialization::write(writer, vec.vec);
}
@ -71,7 +71,7 @@ inline void read(storage::tar::FileReader &reader,
const std::string &name,
detail::PackedVector<T, Bits, Ownership> &vec)
{
vec.num_elements = reader.ReadOne<std::uint64_t>(name + "/number_of_elements.meta");
reader.ReadInto(name + "/number_of_elements.meta", vec.num_elements);
storage::serialization::read(reader, name + "/packed", vec.vec);
}
@ -80,7 +80,7 @@ inline void write(storage::tar::FileWriter &writer,
const std::string &name,
const detail::PackedVector<T, Bits, Ownership> &vec)
{
writer.WriteOne(name + "/number_of_elements.meta", vec.num_elements);
writer.WriteFrom(name + "/number_of_elements.meta", vec.num_elements);
storage::serialization::write(writer, name + "/packed", vec.vec);
}
@ -108,7 +108,7 @@ inline void read(storage::io::FileReader &reader, DynamicGraph<EdgeDataT> &graph
graph.edge_list.resize(num_edges);
for (auto index : irange<std::size_t>(0, num_edges))
{
reader.ReadOne(graph.edge_list[index]);
reader.ReadInto(graph.edge_list[index]);
}
graph.number_of_nodes = graph.node_array.size();
graph.number_of_edges = num_edges;
@ -121,7 +121,7 @@ inline void write(storage::io::FileWriter &writer, const DynamicGraph<EdgeDataT>
writer.WriteElementCount64(graph.number_of_edges);
for (auto index : irange<std::size_t>(0, graph.number_of_edges))
{
writer.WriteOne(graph.edge_list[index]);
writer.WriteFrom(graph.edge_list[index]);
}
}

4
include/util/static_rtree.hpp
View File

@ -452,10 +452,10 @@ class StaticRTree
std::uint64_t size_of_tree = m_search_tree.size();
BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty");
tree_node_file.WriteOne(size_of_tree);
tree_node_file.WriteFrom(size_of_tree);
tree_node_file.WriteFrom(m_search_tree);
tree_node_file.WriteOne(static_cast<std::uint64_t>(m_tree_level_sizes.size()));
tree_node_file.WriteFrom(static_cast<std::uint64_t>(m_tree_level_sizes.size()));
tree_node_file.WriteFrom(m_tree_level_sizes);
}

43
src/extractor/edge_based_graph_factory.cpp
View File

@ -535,21 +535,21 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
&scripting_environment,
weight_multiplier,
&conditional_restriction_map](
// what nodes will be used? In most cases this will be the id
// stored in the edge_data. In case of duplicated nodes (e.g.
// due to via-way restrictions), one/both of these might
// refer to a newly added edge based node
const auto edge_based_node_from,
const auto edge_based_node_to,
// the situation of the turn
const auto node_along_road_entering,
const auto node_based_edge_from,
const auto intersection_node,
const auto node_based_edge_to,
const auto &turn_angle,
const auto &road_legs_on_the_right,
const auto &road_legs_on_the_left,
const auto &edge_geometries) {
// what nodes will be used? In most cases this will be the id
// stored in the edge_data. In case of duplicated nodes (e.g.
// due to via-way restrictions), one/both of these might
// refer to a newly added edge based node
const auto edge_based_node_from,
const auto edge_based_node_to,
// the situation of the turn
const auto node_along_road_entering,
const auto node_based_edge_from,
const auto intersection_node,
const auto node_based_edge_to,
const auto &turn_angle,
const auto &road_legs_on_the_right,
const auto &road_legs_on_the_left,
const auto &edge_geometries) {
const auto node_restricted =
isRestricted(node_along_road_entering,
@ -871,7 +871,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
// TODO: this loop is not optimized - once we have a few
// overrides available, we should index this for faster
// lookups
for (auto & override : unresolved_maneuver_overrides)
for (auto &override : unresolved_maneuver_overrides)
{
for (auto &turn : override.turn_sequence)
{
@ -1127,14 +1127,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
// do not really have a choice but to index the conditional penalties and walk over all
// edge-based-edges to find the ID of the edge
auto const indexed_conditionals = IndexConditionals(std::move(conditionals));
{
util::Log() << "Writing " << indexed_conditionals.size()
<< " conditional turn penalties...";
// write conditional turn penalties into the restrictions file
storage::io::FileWriter writer(conditional_penalties_filename,
storage::io::FileWriter::GenerateFingerprint);
extractor::serialization::write(writer, indexed_conditionals);
}
util::Log() << "Writing " << indexed_conditionals.size() << " conditional turn penalties...";
extractor::files::writeConditionalRestrictions(conditional_penalties_filename,
indexed_conditionals);
// write weight penalties per turn
BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size());

2
src/extractor/extraction_containers.cpp
View File

@ -616,7 +616,7 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
}
BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
file_out.WriteOne((*node_iterator));
file_out.WriteFrom((*node_iterator));
++node_id_iterator;
++node_iterator;

8
src/extractor/extractor.cpp
View File

@ -853,16 +853,16 @@ void Extractor::WriteCompressedNodeBasedGraph(const std::string &path,
{
const auto to_node = graph.GetTarget(edge);
writer.WriteOne(from_node);
writer.WriteOne(to_node);
writer.WriteFrom(from_node);
writer.WriteFrom(to_node);
}
}
// FIXME this is unneccesary: We have this data
for (const auto &qnode : coordinates)
{
writer.WriteOne(qnode.lon);
writer.WriteOne(qnode.lat);
writer.WriteFrom(qnode.lon);
writer.WriteFrom(qnode.lat);
}
}

5
src/updater/updater.cpp
View File

@ -596,9 +596,8 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
std::vector<extractor::ConditionalTurnPenalty> conditional_turns;
if (update_conditional_turns)
{
using storage::io::FileReader;
FileReader reader(config.GetPath(".osrm.restrictions"), FileReader::VerifyFingerprint);
extractor::serialization::read(reader, conditional_turns);
extractor::files::readConditionalRestrictions(config.GetPath(".osrm.restrictions"),
conditional_turns);
}
tbb::concurrent_vector<GeometryID> updated_segments;

16
unit_tests/storage/tar.cpp
View File

@ -51,7 +51,7 @@ BOOST_AUTO_TEST_CASE(read_tar_file)
BOOST_AUTO_TEST_CASE(write_tar_file)
{
TemporaryFile tmp {TEST_DATA_DIR "/tar_write_test.tar"};
TemporaryFile tmp{TEST_DATA_DIR "/tar_write_test.tar"};
std::uint64_t single_64bit_integer = 0xDEADBEEFAABBCCDD;
std::uint32_t single_32bit_integer = 0xDEADBEEF;
@ -62,8 +62,8 @@ BOOST_AUTO_TEST_CASE(write_tar_file)
{
storage::tar::FileWriter writer(tmp.path, storage::tar::FileWriter::GenerateFingerprint);
writer.WriteOne("foo/single_64bit_integer", single_64bit_integer);
writer.WriteOne("bar/single_32bit_integer", single_32bit_integer);
writer.WriteFrom("foo/single_64bit_integer", single_64bit_integer);
writer.WriteFrom("bar/single_32bit_integer", single_32bit_integer);
writer.WriteElementCount64("baz/bla/64bit_vector", vector_64bit.size());
writer.WriteFrom("baz/bla/64bit_vector", vector_64bit.data(), vector_64bit.size());
writer.WriteElementCount64("32bit_vector", vector_32bit.size());
@ -72,10 +72,12 @@ BOOST_AUTO_TEST_CASE(write_tar_file)
storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
BOOST_CHECK_EQUAL(reader.ReadOne<std::uint32_t>("bar/single_32bit_integer"),
single_32bit_integer);
BOOST_CHECK_EQUAL(reader.ReadOne<std::uint64_t>("foo/single_64bit_integer"),
single_64bit_integer);
std::uint32_t result_32bit_integer;
std::uint64_t result_64bit_integer;
reader.ReadInto("bar/single_32bit_integer", result_32bit_integer);
reader.ReadInto("foo/single_64bit_integer", result_64bit_integer);
BOOST_CHECK_EQUAL(result_32bit_integer, single_32bit_integer);
BOOST_CHECK_EQUAL(result_32bit_integer, single_64bit_integer);
std::vector<std::uint64_t> result_64bit_vector(
reader.ReadElementCount64("baz/bla/64bit_vector"));

4
unit_tests/util/io.cpp
View File

@ -123,7 +123,7 @@ BOOST_AUTO_TEST_CASE(io_corrupt_fingerprint)
osrm::storage::io::FileWriter outfile(IO_CORRUPT_FINGERPRINT_FILE,
osrm::storage::io::FileWriter::HasNoFingerprint);
outfile.WriteOne(0xDEADBEEFCAFEFACE);
outfile.WriteFrom(0xDEADBEEFCAFEFACE);
osrm::storage::serialization::write(outfile, v);
}
@ -154,7 +154,7 @@ BOOST_AUTO_TEST_CASE(io_incompatible_fingerprint)
osrm::storage::io::FileWriter::HasNoFingerprint);
const auto fingerprint = osrm::util::FingerPrint::GetValid();
outfile.WriteOne(fingerprint);
outfile.WriteFrom(fingerprint);
osrm::storage::serialization::write(outfile, v);
}