Add BufferWriter/BufferReader and simplify interface for ConditionalRestrictions

include/contractor/files.hpp

@@ -32,7 +32,7 @@ inline void readGraph(const boost::filesystem::path &path,
     const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
     storage::tar::FileReader reader{path, fingerprint};
 
-    checksum = reader.ReadOne<std::uint32_t>("/ch/checksum");
+    reader.ReadInto("/ch/checksum", checksum);
     util::serialization::read(reader, "/ch/contracted_graph", graph);
 
     auto count = reader.ReadElementCount64("/ch/edge_filter");
@@ -42,7 +42,7 @@ inline void readGraph(const boost::filesystem::path &path,
         storage::serialization::read(reader, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
     }
 
-    connectivity_checksum = reader.ReadOne<std::uint32_t>("/ch/connectivity_checksum");
+     reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum);
 }
 
 // writes .osrm.hsgr file
@@ -63,7 +63,7 @@ inline void writeGraph(const boost::filesystem::path &path,
     storage::tar::FileWriter writer{path, fingerprint};
 
     writer.WriteElementCount64("/ch/checksum", 1);
-    writer.WriteOne("/ch/checksum", checksum);
+    writer.WriteFrom("/ch/checksum", checksum);
     util::serialization::write(writer, "/ch/contracted_graph", graph);
 
     writer.WriteElementCount64("/ch/edge_filter", edge_filter.size());
@@ -73,7 +73,7 @@ inline void writeGraph(const boost::filesystem::path &path,
     }
 
     writer.WriteElementCount64("/ch/connectivity_checksum", 1);
-    writer.WriteOne("/ch/connectivity_checksum", connectivity_checksum);
+    writer.WriteFrom("/ch/connectivity_checksum", connectivity_checksum);
 }
 }
 }

include/extractor/files.hpp

@@ -87,10 +87,10 @@ void writeEdgeBasedGraph(const boost::filesystem::path &path,
     storage::tar::FileWriter writer(path, storage::tar::FileWriter::GenerateFingerprint);
 
     writer.WriteElementCount64("/common/number_of_edge_based_nodes", 1);
-    writer.WriteOne("/common/number_of_edge_based_nodes", number_of_edge_based_nodes);
+    writer.WriteFrom("/common/number_of_edge_based_nodes", number_of_edge_based_nodes);
     storage::serialization::write(writer, "/common/edge_based_edge_list", edge_based_edge_list);
     writer.WriteElementCount64("/common/connectivity_checksum", 1);
-    writer.WriteOne("/common/connectivity_checksum", connectivity_checksum);
+    writer.WriteFrom("/common/connectivity_checksum", connectivity_checksum);
 }
 
 template <typename EdgeBasedEdgeVector>
@@ -103,9 +103,9 @@ void readEdgeBasedGraph(const boost::filesystem::path &path,
 
     storage::tar::FileReader reader(path, storage::tar::FileReader::VerifyFingerprint);
 
-    number_of_edge_based_nodes = reader.ReadOne<EdgeID>("/common/number_of_edge_based_nodes");
+    reader.ReadInto("/common/number_of_edge_based_nodes", number_of_edge_based_nodes);
     storage::serialization::read(reader, "/common/edge_based_edge_list", edge_based_edge_list);
-    connectivity_checksum = reader.ReadOne<std::uint32_t>("/common/connectivity_checksum");
+    reader.ReadInto("/common/connectivity_checksum", connectivity_checksum);
 }
 
 // reads .osrm.nbg_nodes
@@ -364,6 +364,27 @@ inline void readTurnDurationPenalty(const boost::filesystem::path &path, TurnPen
 
     storage::serialization::read(reader, "/common/turn_penalty/duration", turn_penalty);
 }
+
+// writes .osrm.restrictions
+template <typename ConditionalRestrictionsT>
+inline void writeConditionalRestrictions(const boost::filesystem::path &path,
+                                     const ConditionalRestrictionsT &conditional_restrictions)
+{
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
+
+    serialization::write(writer, "/common/conditional_restrictions", conditional_restrictions);
+}
+
+// read .osrm.restrictions
+template <typename ConditionalRestrictionsT>
+inline void readConditionalRestrictions(const boost::filesystem::path &path, ConditionalRestrictionsT &conditional_restrictions)
+{
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
+
+    serialization::read(reader, "/common/conditional_restrictions", conditional_restrictions);
+}
 }
 }
 }

include/extractor/serialization.hpp

@@ -52,7 +52,7 @@ inline void write(storage::tar::FileWriter &writer,
 inline void
 read(storage::tar::FileReader &reader, const std::string &name, ProfileProperties &properties)
 {
-    properties = reader.ReadOne<ProfileProperties>(name);
+    reader.ReadInto(name, properties);
 }
 
 inline void write(storage::tar::FileWriter &writer,
@@ -60,19 +60,19 @@ inline void write(storage::tar::FileWriter &writer,
                   const ProfileProperties &properties)
 {
     writer.WriteElementCount64(name, 1);
-    writer.WriteOne(name, properties);
+    writer.WriteFrom(name, properties);
 }
 
 // read/write for datasources file
 inline void read(storage::tar::FileReader &reader, const std::string &name, Datasources &sources)
 {
-    sources = reader.ReadOne<Datasources>(name);
+    reader.ReadInto(name, sources);
 }
 
 inline void write(storage::tar::FileWriter &writer, const std::string &name, Datasources &sources)
 {
     writer.WriteElementCount64(name, 1);
-    writer.WriteOne(name, sources);
+    writer.WriteFrom(name, sources);
 }
 
 // read/write for segment data file
@@ -138,9 +138,9 @@ inline void read(storage::io::FileReader &reader, NodeRestriction &restriction)
 
 inline void write(storage::io::FileWriter &writer, const NodeRestriction &restriction)
 {
-    writer.WriteOne(restriction.from);
+    writer.WriteFrom(restriction.from);
-    writer.WriteOne(restriction.via);
+    writer.WriteFrom(restriction.via);
-    writer.WriteOne(restriction.to);
+    writer.WriteFrom(restriction.to);
 }
 
 inline void read(storage::io::FileReader &reader, WayRestriction &restriction)
@@ -177,9 +177,9 @@ inline void read(storage::io::FileReader &reader, TurnRestriction &restriction)
 
 inline void write(storage::io::FileWriter &writer, const TurnRestriction &restriction)
 {
-    writer.WriteOne(restriction.is_only);
+    writer.WriteFrom(restriction.is_only);
     const std::uint32_t restriction_type = restriction.Type();
-    writer.WriteOne(restriction_type);
+    writer.WriteFrom(restriction_type);
     if (restriction.Type() == RestrictionType::WAY_RESTRICTION)
     {
         write(writer, mapbox::util::get<WayRestriction>(restriction.node_or_way));
@@ -197,7 +197,7 @@ inline void write(storage::io::FileWriter &writer, const ConditionalTurnRestrict
     writer.WriteElementCount64(restriction.condition.size());
     for (const auto &c : restriction.condition)
     {
-        writer.WriteOne(c.modifier);
+        writer.WriteFrom(c.modifier);
         storage::serialization::write(writer, c.times);
         storage::serialization::write(writer, c.weekdays);
         storage::serialization::write(writer, c.monthdays);
@@ -266,7 +266,7 @@ inline void write(storage::io::FileWriter &writer,
     std::for_each(restrictions.begin(), restrictions.end(), write_restriction);
 }
 
-inline void read(storage::io::FileReader &reader, ConditionalTurnPenalty &turn_penalty)
+inline void read(storage::io::BufferReader &reader, ConditionalTurnPenalty &turn_penalty)
 {
     reader.ReadInto(turn_penalty.turn_offset);
     reader.ReadInto(turn_penalty.location.lat);
@@ -282,36 +282,61 @@ inline void read(storage::io::FileReader &reader, ConditionalTurnPenalty &turn_p
     }
 }
 
-inline void write(storage::io::FileWriter &writer, const ConditionalTurnPenalty &turn_penalty)
+inline void write(storage::io::BufferWriter &writer, const ConditionalTurnPenalty &turn_penalty)
 {
-    writer.WriteOne(turn_penalty.turn_offset);
+    writer.WriteFrom(turn_penalty.turn_offset);
-    writer.WriteOne(static_cast<util::FixedLatitude::value_type>(turn_penalty.location.lat));
+    writer.WriteFrom(static_cast<util::FixedLatitude::value_type>(turn_penalty.location.lat));
-    writer.WriteOne(static_cast<util::FixedLongitude::value_type>(turn_penalty.location.lon));
+    writer.WriteFrom(static_cast<util::FixedLongitude::value_type>(turn_penalty.location.lon));
     writer.WriteElementCount64(turn_penalty.conditions.size());
     for (const auto &c : turn_penalty.conditions)
     {
-        writer.WriteOne(c.modifier);
+        writer.WriteFrom(c.modifier);
         storage::serialization::write(writer, c.times);
         storage::serialization::write(writer, c.weekdays);
         storage::serialization::write(writer, c.monthdays);
     }
 }
 
-inline void write(storage::io::FileWriter &writer,
+inline void write(storage::io::BufferWriter &writer,
                   const std::vector<ConditionalTurnPenalty> &conditional_penalties)
 {
     writer.WriteElementCount64(conditional_penalties.size());
     for (const auto &penalty : conditional_penalties)
+    {
         write(writer, penalty);
+    }
 }
 
-inline void read(storage::io::FileReader &reader,
+inline void read(storage::io::BufferReader &reader, std::vector<ConditionalTurnPenalty> &conditional_penalties)
-                 std::vector<ConditionalTurnPenalty> &conditional_penalties)
 {
-    auto const num_elements = reader.ReadElementCount64();
+    auto num_elements = reader.ReadElementCount64();
     conditional_penalties.resize(num_elements);
     for (auto &penalty : conditional_penalties)
+    {
         read(reader, penalty);
+    }
+}
+
+inline void write(storage::tar::FileWriter &writer,
+                 const std::string& name,
+                 const std::vector<ConditionalTurnPenalty> &conditional_penalties)
+{
+    storage::io::BufferWriter buffer_writer;
+    write(buffer_writer, conditional_penalties);
+
+    storage::serialization::write(writer, name, buffer_writer.GetBuffer());
+}
+
+inline void read(storage::tar::FileReader &reader,
+                 const std::string& name,
+                 std::vector<ConditionalTurnPenalty> &conditional_penalties)
+{
+    std::string buffer;
+    storage::serialization::read(reader, name, buffer);
+
+    storage::io::BufferReader buffer_reader{buffer};
+    read(buffer_reader, conditional_penalties);
+
 }
 
 }

include/guidance/serialization.hpp

@@ -39,7 +39,7 @@ inline void write(storage::io::FileWriter &writer,
     storage::serialization::write(writer, turn_data_container.entry_class_ids);
     storage::serialization::write(writer, turn_data_container.pre_turn_bearings);
     storage::serialization::write(writer, turn_data_container.post_turn_bearings);
-    writer.WriteOne(connectivity_checksum);
+    writer.WriteFrom(connectivity_checksum);
 }
 }
 }

include/partitioner/serialization.hpp

@@ -28,7 +28,7 @@ inline void read(storage::tar::FileReader &reader,
     storage::serialization::read(reader, name + "/node_array", graph.node_array);
     storage::serialization::read(reader, name + "/edge_array", graph.edge_array);
     storage::serialization::read(reader, name + "/node_to_edge_offset", graph.node_to_edge_offset);
-    connectivity_checksum = reader.ReadOne<std::uint32_t>(name + "/connectivity_checksum");
+    reader.ReadInto(name + "/connectivity_checksum", connectivity_checksum);
 }
 
 template <typename EdgeDataT, storage::Ownership Ownership>
@@ -41,7 +41,7 @@ inline void write(storage::tar::FileWriter &writer,
     storage::serialization::write(writer, name + "/edge_array", graph.edge_array);
     storage::serialization::write(writer, name + "/node_to_edge_offset", graph.node_to_edge_offset);
     writer.WriteElementCount64(name + "/connectivity_checksum", 1);
-    writer.WriteOne(name + "/connectivity_checksum", connectivity_checksum);
+    writer.WriteFrom(name + "/connectivity_checksum", connectivity_checksum);
 }
 
 template <storage::Ownership Ownership>
@@ -49,7 +49,7 @@ inline void read(storage::tar::FileReader &reader,
                  const std::string &name,
                  detail::MultiLevelPartitionImpl<Ownership> &mlp)
 {
-    *mlp.level_data = reader.ReadOne<typename std::remove_reference_t<decltype(mlp)>::LevelData>(name + "/level_data");
+    reader.ReadInto(name + "/level_data", *mlp.level_data);
     storage::serialization::read(reader, name + "/partition", mlp.partition);
     storage::serialization::read(reader, name + "/cell_to_children", mlp.cell_to_children);
 }
@@ -60,7 +60,7 @@ inline void write(storage::tar::FileWriter &writer,
                   const detail::MultiLevelPartitionImpl<Ownership> &mlp)
 {
     writer.WriteElementCount64(name + "/level_data", 1);
-    writer.WriteOne(name + "/level_data", *mlp.level_data);
+    writer.WriteFrom(name + "/level_data", *mlp.level_data);
     storage::serialization::write(writer, name + "/partition", mlp.partition);
     storage::serialization::write(writer, name + "/cell_to_children", mlp.cell_to_children);
 }

include/storage/io.hpp

@@ -6,6 +6,7 @@
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
 #include "util/fingerprint.hpp"
+#include "util/integer_range.hpp"
 #include "util/log.hpp"
 #include "util/version.hpp"
 
@@ -110,6 +111,38 @@ class FileReader
         }
     }
 
+    template <typename T, typename OutIter> void ReadStreaming(OutIter out, const std::size_t count)
+    {
+#if !defined(__GNUC__) || (__GNUC__ > 4)
+        static_assert(!std::is_pointer<T>::value, "saving pointer types is not allowed");
+        static_assert(std::is_trivially_copyable<T>::value,
+                      "bytewise reading requires trivially copyable type");
+#endif
+
+        if (count == 0)
+            return;
+
+        T tmp;
+        for (auto index : util::irange<std::size_t>(0, count))
+        {
+            (void)index;
+            const auto &result = input_stream.read(reinterpret_cast<char *>(&tmp), sizeof(T));
+            const std::size_t bytes_read = input_stream.gcount();
+
+            if (bytes_read != sizeof(T) && !result)
+            {
+                if (result.eof())
+                {
+                    throw util::RuntimeError(
+                        filepath.string(), ErrorCode::UnexpectedEndOfFile, SOURCE_REF);
+                }
+                throw util::RuntimeError(
+                    filepath.string(), ErrorCode::FileReadError, SOURCE_REF, std::strerror(errno));
+            }
+            *out++ = tmp;
+        }
+    }
+
     template <typename T> void ReadInto(std::vector<T> &target)
     {
         ReadInto(target.data(), target.size());
@@ -117,13 +150,6 @@ class FileReader
 
     template <typename T> void ReadInto(T &target) { ReadInto(&target, 1); }
 
-    template <typename T> T ReadOne()
-    {
-        T tmp;
-        ReadInto(tmp);
-        return tmp;
-    }
-
     template <typename T> void Skip(const std::size_t element_count)
     {
         boost::iostreams::seek(input_stream, element_count * sizeof(T), BOOST_IOS::cur);
@@ -131,7 +157,12 @@ class FileReader
 
     /*******************************************/
 
-    std::uint64_t ReadElementCount64() { return ReadOne<std::uint64_t>(); }
+    std::uint64_t ReadElementCount64()
+    {
+        std::uint64_t count;
+        ReadInto(count);
+        return count;
+    }
 
     template <typename T> std::size_t ReadVectorSize()
     {
@@ -142,7 +173,8 @@ class FileReader
 
     bool ReadAndCheckFingerprint()
     {
-        auto loaded_fingerprint = ReadOne<util::FingerPrint>();
+        util::FingerPrint loaded_fingerprint;
+        ReadInto(loaded_fingerprint);
         const auto expected_fingerprint = util::FingerPrint::GetValid();
 
         if (!loaded_fingerprint.IsValid())
@@ -229,14 +261,12 @@ class FileWriter
 
     template <typename T> void WriteFrom(const T &src) { WriteFrom(&src, 1); }
 
-    template <typename T> void WriteOne(const T &tmp) { WriteFrom(tmp); }
+    void WriteElementCount64(const std::uint64_t count) { WriteFrom(count); }
-
-    void WriteElementCount64(const std::uint64_t count) { WriteOne<std::uint64_t>(count); }
 
     void WriteFingerprint()
     {
         const auto fingerprint = util::FingerPrint::GetValid();
-        return WriteOne(fingerprint);
+        return WriteFrom(fingerprint);
     }
 
     template <typename T> void Skip(const std::size_t element_count)
@@ -298,14 +328,14 @@ class BufferReader
         }
     }
 
-    template <typename T> T ReadOne()
+    template <typename T> void ReadInto(T &tmp) { ReadInto(&tmp, 1); }
-    {
-        T tmp;
-        ReadInto(&tmp, 1);
-        return tmp;
-    }
 
-    std::uint64_t ReadElementCount64() { return ReadOne<std::uint64_t>(); }
+    std::uint64_t ReadElementCount64()
+    {
+        std::uint64_t count;
+        ReadInto(count);
+        return count;
+    }
 
   private:
     std::istringstream input_stream;
@@ -343,9 +373,9 @@ class BufferWriter
         }
     }
 
-    template <typename T> void WriteOne(const T &tmp) { WriteFrom(&tmp, 1); }
+    template <typename T> void WriteFrom(const T &tmp) { WriteFrom(&tmp, 1); }
 
-    void WriteElementCount64(const std::uint64_t count) { WriteOne<std::uint64_t>(count); }
+    void WriteElementCount64(const std::uint64_t count) { WriteFrom(count); }
 
     std::string GetBuffer() const { return output_stream.str(); }
 

include/storage/serialization.hpp

@@ -84,22 +84,24 @@ inline void write(storage::tar::FileWriter &writer, const std::string& name, con
 #if USE_STXXL_LIBRARY
 template <typename T> inline void read(storage::io::FileReader &reader, stxxl::vector<T> &vec)
 {
-    auto size = reader.ReadOne<std::uint64_t>();
+    auto size = reader.ReadElementCount64();
     vec.reserve(size);
+    T tmp;
     for (auto idx : util::irange<std::size_t>(0, size))
     {
         (void)idx;
-        vec.push_back(reader.ReadOne<T>());
+        reader.ReadInto(tmp);
+        vec.push_back(tmp);
     }
 }
 
 template <typename T>
 inline void write(storage::io::FileWriter &writer, const stxxl::vector<T> &vec)
 {
-    writer.WriteOne(vec.size());
+    writer.WriteFrom(vec.size());
     for (auto idx : util::irange<std::size_t>(0, vec.size()))
     {
-        writer.WriteOne<T>(vec[idx]);
+        writer.WriteFrom<T>(vec[idx]);
     }
 }
 #endif
@@ -120,8 +122,22 @@ void write(io::BufferWriter &writer, const std::vector<T> &data)
     writer.WriteFrom(data.data(), count);
 }
 
+inline void write(tar::FileWriter &writer, const std::string &name, const std::string &data)
+{
+    const auto count = data.size();
+    writer.WriteElementCount64(name, count);
+    writer.WriteFrom(name, data.data(), count);
+}
+
+inline void read(tar::FileReader &reader, const std::string &name, std::string &data)
+{
+    const auto count = reader.ReadElementCount64(name);
+    data.resize(count);
+    reader.ReadInto(name, const_cast<char*>(data.data()), count);
+}
+
 template <typename T>
-void read(tar::FileReader &reader, const std::string &name, std::vector<T> &data)
+inline void read(tar::FileReader &reader, const std::string& name, std::vector<T> &data)
 {
     const auto count = reader.ReadElementCount64(name);
     data.resize(count);
@@ -140,7 +156,7 @@ template <typename T>
 void read(tar::FileReader &reader, const std::string &name, util::vector_view<T> &data)
 {
     const auto count = reader.ReadElementCount64(name);
-    BOOST_ASSERT(data.size() == count);
+    data.resize(count);
     reader.ReadInto(name, data.data(), count);
 }
 
@@ -206,12 +222,17 @@ template <typename VectorT> void readBoolVector(io::FileReader &reader, VectorT
     const auto count = reader.ReadElementCount64();
     data.resize(count);
     std::uint64_t index = 0;
+    unsigned char block_data;
     for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
     {
-        unpackBits(data, index, CHAR_BIT, reader.ReadOne<unsigned char>());
+        reader.ReadInto(block_data);
+        unpackBits(data, index, CHAR_BIT, block_data);
     }
     if (count > index)
-        unpackBits(data, index, count - index, reader.ReadOne<unsigned char>());
+    {
+        reader.ReadInto(block_data);
+        unpackBits(data, index, count - index, block_data);
+    }
 }
 
 template <typename VectorT> void writeBoolVector(io::FileWriter &writer, const VectorT &data)
@@ -221,10 +242,10 @@ template <typename VectorT> void writeBoolVector(io::FileWriter &writer, const V
     std::uint64_t index = 0;
     for (std::uint64_t next = CHAR_BIT; next < count; index = next, next += CHAR_BIT)
     {
-        writer.WriteOne<unsigned char>(packBits(data, index, CHAR_BIT));
+        writer.WriteFrom<unsigned char>(packBits(data, index, CHAR_BIT));
     }
     if (count > index)
-        writer.WriteOne<unsigned char>(packBits(data, index, count - index));
+        writer.WriteFrom<unsigned char>(packBits(data, index, count - index));
 }
 
 template <typename VectorT>

include/storage/tar.hpp

@@ -47,14 +47,14 @@ class FileReader
 
     std::uint64_t ReadElementCount64(const std::string &name)
     {
-        return ReadOne<std::uint64_t>(name + ".meta");
+        std::uint64_t size;
+        ReadInto(name + ".meta", size);
+        return size;
     }
 
-    template <typename T> T ReadOne(const std::string &name)
+    template <typename T> void ReadInto(const std::string &name, T& tmp)
     {
-        T tmp;
         ReadInto(name, &tmp, 1);
-        return tmp;
     }
 
     template <typename T, typename OutIter> void ReadStreaming(const std::string &name, OutIter out)
@@ -127,7 +127,8 @@ class FileReader
   private:
     bool ReadAndCheckFingerprint()
     {
-        auto loaded_fingerprint = ReadOne<util::FingerPrint>("osrm_fingerprint.meta");
+        util::FingerPrint loaded_fingerprint;
+        ReadInto("osrm_fingerprint.meta", loaded_fingerprint);
         const auto expected_fingerprint = util::FingerPrint::GetValid();
 
         if (!loaded_fingerprint.IsValid())
@@ -183,10 +184,10 @@ class FileWriter
 
     void WriteElementCount64(const std::string &name, const std::uint64_t count)
     {
-        WriteOne(name + ".meta", count);
+        WriteFrom(name + ".meta", count);
     }
 
-    template <typename T> void WriteOne(const std::string &name, const T &data)
+    template <typename T> void WriteFrom(const std::string &name, const T &data)
     {
         WriteFrom(name, &data, 1);
     }
@@ -236,7 +237,7 @@ class FileWriter
     void WriteFingerprint()
     {
         const auto fingerprint = util::FingerPrint::GetValid();
-        WriteOne("osrm_fingerprint.meta", fingerprint);
+        WriteFrom("osrm_fingerprint.meta", fingerprint);
     }
 
     boost::filesystem::path path;

include/util/graph_loader.hpp

@@ -15,6 +15,7 @@
 #include <boost/assert.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
+#include <boost/function_output_iterator.hpp>
 
 #include <tbb/parallel_sort.h>
 
@@ -49,27 +50,22 @@ NodeID loadNodesFromFile(storage::io::FileReader &file_reader,
     coordinates.resize(number_of_nodes);
     osm_node_ids.reserve(number_of_nodes);
 
-    extractor::QueryNode current_node;
+    auto index = 0;
-    for (NodeID i = 0; i < number_of_nodes; ++i)
+    auto decode = [&](const extractor::QueryNode &current_node) {
-    {
+        coordinates[index].lon = current_node.lon;
-        file_reader.ReadInto(&current_node, 1);
+        coordinates[index].lat = current_node.lat;
-
-        coordinates[i].lon = current_node.lon;
-        coordinates[i].lat = current_node.lat;
         osm_node_ids.push_back(current_node.node_id);
-    }
+        index++;
+    };
+
+    file_reader.ReadStreaming<extractor::QueryNode>(boost::make_function_output_iterator(decode),
+                                                    number_of_nodes);
 
     auto num_barriers = file_reader.ReadElementCount64();
-    for (auto index = 0UL; index < num_barriers; ++index)
+    file_reader.ReadStreaming<NodeID>(barriers, num_barriers);
-    {
-        *barriers++ = file_reader.ReadOne<NodeID>();
-    }
 
     auto num_lights = file_reader.ReadElementCount64();
-    for (auto index = 0UL; index < num_lights; ++index)
+    file_reader.ReadStreaming<NodeID>(traffic_signals, num_lights);
-    {
-        *traffic_signals++ = file_reader.ReadOne<NodeID>();
-    }
 
     return number_of_nodes;
 }

include/util/indexed_data.hpp

@@ -106,7 +106,7 @@ template <int N, typename T = std::string> struct VariableGroupBlock
             prefix_length += byte_length;
         }
 
-        out.WriteOne(refernce);
+        out.WriteFrom(refernce);
 
         return prefix_length;
     }
@@ -187,7 +187,7 @@ template <int N, typename T = std::string> struct FixedGroupBlock
         BOOST_ASSERT(data_offset <= std::numeric_limits<decltype(BlockReference::offset)>::max());
 
         BlockReference refernce{static_cast<decltype(BlockReference::offset)>(data_offset)};
-        out.WriteOne(refernce);
+        out.WriteFrom(refernce);
 
         return BLOCK_SIZE;
     }
@@ -268,7 +268,7 @@ template <typename GroupBlock> struct IndexedData
         const BlocksNumberType number_of_blocks =
             number_of_elements == 0 ? 0
                                     : 1 + (std::distance(first, sentinel) - 1) / (BLOCK_SIZE + 1);
-        out.WriteOne(number_of_blocks);
+        out.WriteFrom(number_of_blocks);
 
         // Write block references and compute the total data size that includes prefix and data
         const GroupBlock block;
@@ -282,7 +282,7 @@ template <typename GroupBlock> struct IndexedData
         }
 
         // Write the total data size
-        out.WriteOne(data_size);
+        out.WriteFrom(data_size);
 
         // Write data blocks that are (prefix, data)
         for (OffsetIterator curr = first, next = first; next != sentinel; curr = next)
@@ -291,7 +291,7 @@ template <typename GroupBlock> struct IndexedData
             block.WriteBlockPrefix(out, curr, next);
             std::advance(next, std::min<diff_type>(1, std::distance(next, sentinel)));
             std::for_each(
-                data + *curr, data + *next, [&out](const auto &element) { out.WriteOne(element); });
+                data + *curr, data + *next, [&out](const auto &element) { out.WriteFrom(element); });
         }
     }
 

include/util/serialization.hpp

@@ -18,7 +18,7 @@ namespace serialization
 template <unsigned BlockSize, storage::Ownership Ownership>
 void write(storage::io::FileWriter &writer, const util::RangeTable<BlockSize, Ownership> &table)
 {
-    writer.WriteOne(table.sum_lengths);
+    writer.WriteFrom(table.sum_lengths);
     storage::serialization::write(writer, table.block_offsets);
     storage::serialization::write(writer, table.diff_blocks);
 }
@@ -26,7 +26,7 @@ void write(storage::io::FileWriter &writer, const util::RangeTable<BlockSize, Ow
 template <unsigned BlockSize, storage::Ownership Ownership>
 void read(storage::io::FileReader &reader, util::RangeTable<BlockSize, Ownership> &table)
 {
-    table.sum_lengths = reader.ReadOne<unsigned>();
+    reader.ReadInto(table.sum_lengths);
     storage::serialization::read(reader, table.block_offsets);
     storage::serialization::read(reader, table.diff_blocks);
 }
@@ -36,7 +36,7 @@ void write(storage::tar::FileWriter &writer,
            const std::string &name,
            const util::RangeTable<BlockSize, Ownership> &table)
 {
-    writer.WriteOne(name + "/sum_lengths.meta", table.sum_lengths);
+    writer.WriteFrom(name + "/sum_lengths.meta", table.sum_lengths);
     storage::serialization::write(writer, name + "/block_offsets", table.block_offsets);
     storage::serialization::write(writer, name + "/diff_blocks", table.diff_blocks);
 }
@@ -46,7 +46,7 @@ void read(storage::tar::FileReader &reader,
           const std::string &name,
           util::RangeTable<BlockSize, Ownership> &table)
 {
-    table.sum_lengths = reader.ReadOne<unsigned>(name + "/sum_lengths.meta");
+    reader.ReadInto(name + "/sum_lengths.meta", table.sum_lengths);
     storage::serialization::read(reader, name + "/block_offsets", table.block_offsets);
     storage::serialization::read(reader, name + "/diff_blocks", table.diff_blocks);
 }
@@ -54,7 +54,7 @@ void read(storage::tar::FileReader &reader,
 template <typename T, std::size_t Bits, storage::Ownership Ownership>
 inline void read(storage::io::FileReader &reader, detail::PackedVector<T, Bits, Ownership> &vec)
 {
-    vec.num_elements = reader.ReadOne<std::uint64_t>();
+    reader.ReadInto(vec.num_elements);
     storage::serialization::read(reader, vec.vec);
 }
 
@@ -62,7 +62,7 @@ template <typename T, std::size_t Bits, storage::Ownership Ownership>
 inline void write(storage::io::FileWriter &writer,
                   const detail::PackedVector<T, Bits, Ownership> &vec)
 {
-    writer.WriteOne(vec.num_elements);
+    writer.WriteFrom(vec.num_elements);
     storage::serialization::write(writer, vec.vec);
 }
 
@@ -71,7 +71,7 @@ inline void read(storage::tar::FileReader &reader,
                  const std::string &name,
                  detail::PackedVector<T, Bits, Ownership> &vec)
 {
-    vec.num_elements = reader.ReadOne<std::uint64_t>(name + "/number_of_elements.meta");
+    reader.ReadInto(name + "/number_of_elements.meta", vec.num_elements);
     storage::serialization::read(reader, name + "/packed", vec.vec);
 }
 
@@ -80,7 +80,7 @@ inline void write(storage::tar::FileWriter &writer,
                   const std::string &name,
                   const detail::PackedVector<T, Bits, Ownership> &vec)
 {
-    writer.WriteOne(name + "/number_of_elements.meta", vec.num_elements);
+    writer.WriteFrom(name + "/number_of_elements.meta", vec.num_elements);
     storage::serialization::write(writer, name + "/packed", vec.vec);
 }
 
@@ -108,7 +108,7 @@ inline void read(storage::io::FileReader &reader, DynamicGraph<EdgeDataT> &graph
     graph.edge_list.resize(num_edges);
     for (auto index : irange<std::size_t>(0, num_edges))
     {
-        reader.ReadOne(graph.edge_list[index]);
+        reader.ReadInto(graph.edge_list[index]);
     }
     graph.number_of_nodes = graph.node_array.size();
     graph.number_of_edges = num_edges;
@@ -121,7 +121,7 @@ inline void write(storage::io::FileWriter &writer, const DynamicGraph<EdgeDataT>
     writer.WriteElementCount64(graph.number_of_edges);
     for (auto index : irange<std::size_t>(0, graph.number_of_edges))
     {
-        writer.WriteOne(graph.edge_list[index]);
+        writer.WriteFrom(graph.edge_list[index]);
     }
 }
 

include/util/static_rtree.hpp

@@ -452,10 +452,10 @@ class StaticRTree
             std::uint64_t size_of_tree = m_search_tree.size();
             BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty");
 
-            tree_node_file.WriteOne(size_of_tree);
+            tree_node_file.WriteFrom(size_of_tree);
             tree_node_file.WriteFrom(m_search_tree);
 
-            tree_node_file.WriteOne(static_cast<std::uint64_t>(m_tree_level_sizes.size()));
+            tree_node_file.WriteFrom(static_cast<std::uint64_t>(m_tree_level_sizes.size()));
             tree_node_file.WriteFrom(m_tree_level_sizes);
         }
 

src/extractor/edge_based_graph_factory.cpp

@@ -535,21 +535,21 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
                                     &scripting_environment,
                                     weight_multiplier,
                                     &conditional_restriction_map](
-            // what nodes will be used? In most cases this will be the id
+                                       // what nodes will be used? In most cases this will be the id
-            // stored in the edge_data. In case of duplicated nodes (e.g.
+                                       // stored in the edge_data. In case of duplicated nodes (e.g.
-            // due to via-way restrictions), one/both of these might
+                                       // due to via-way restrictions), one/both of these might
-            // refer to a newly added edge based node
+                                       // refer to a newly added edge based node
-            const auto edge_based_node_from,
+                                       const auto edge_based_node_from,
-            const auto edge_based_node_to,
+                                       const auto edge_based_node_to,
-            // the situation of the turn
+                                       // the situation of the turn
-            const auto node_along_road_entering,
+                                       const auto node_along_road_entering,
-            const auto node_based_edge_from,
+                                       const auto node_based_edge_from,
-            const auto intersection_node,
+                                       const auto intersection_node,
-            const auto node_based_edge_to,
+                                       const auto node_based_edge_to,
-            const auto &turn_angle,
+                                       const auto &turn_angle,
-            const auto &road_legs_on_the_right,
+                                       const auto &road_legs_on_the_right,
-            const auto &road_legs_on_the_left,
+                                       const auto &road_legs_on_the_left,
-            const auto &edge_geometries) {
+                                       const auto &edge_geometries) {
 
             const auto node_restricted =
                 isRestricted(node_along_road_entering,
@@ -871,7 +871,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
                             // TODO: this loop is not optimized - once we have a few
                             //       overrides available, we should index this for faster
                             //       lookups
-                            for (auto & override : unresolved_maneuver_overrides)
+                            for (auto &override : unresolved_maneuver_overrides)
                             {
                                 for (auto &turn : override.turn_sequence)
                                 {
@@ -1127,14 +1127,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
     // do not really have a choice but to index the conditional penalties and walk over all
     // edge-based-edges to find the ID of the edge
     auto const indexed_conditionals = IndexConditionals(std::move(conditionals));
-    {
+    util::Log() << "Writing " << indexed_conditionals.size() << " conditional turn penalties...";
-        util::Log() << "Writing " << indexed_conditionals.size()
+    extractor::files::writeConditionalRestrictions(conditional_penalties_filename,
-                    << " conditional turn penalties...";
+                                                   indexed_conditionals);
-        // write conditional turn penalties into the restrictions file
-        storage::io::FileWriter writer(conditional_penalties_filename,
-                                       storage::io::FileWriter::GenerateFingerprint);
-        extractor::serialization::write(writer, indexed_conditionals);
-    }
 
     // write weight penalties per turn
     BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size());

src/extractor/extraction_containers.cpp

@@ -616,7 +616,7 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const
             }
             BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
 
-            file_out.WriteOne((*node_iterator));
+            file_out.WriteFrom((*node_iterator));
 
             ++node_id_iterator;
             ++node_iterator;

src/extractor/extractor.cpp

@@ -853,16 +853,16 @@ void Extractor::WriteCompressedNodeBasedGraph(const std::string &path,
         {
             const auto to_node = graph.GetTarget(edge);
 
-            writer.WriteOne(from_node);
+            writer.WriteFrom(from_node);
-            writer.WriteOne(to_node);
+            writer.WriteFrom(to_node);
         }
     }
 
     // FIXME this is unneccesary: We have this data
     for (const auto &qnode : coordinates)
     {
-        writer.WriteOne(qnode.lon);
+        writer.WriteFrom(qnode.lon);
-        writer.WriteOne(qnode.lat);
+        writer.WriteFrom(qnode.lat);
     }
 }
 

src/updater/updater.cpp

@@ -596,9 +596,8 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
     std::vector<extractor::ConditionalTurnPenalty> conditional_turns;
     if (update_conditional_turns)
     {
-        using storage::io::FileReader;
+        extractor::files::readConditionalRestrictions(config.GetPath(".osrm.restrictions"),
-        FileReader reader(config.GetPath(".osrm.restrictions"), FileReader::VerifyFingerprint);
+                                                      conditional_turns);
-        extractor::serialization::read(reader, conditional_turns);
     }
 
     tbb::concurrent_vector<GeometryID> updated_segments;

unit_tests/storage/tar.cpp

@@ -51,7 +51,7 @@ BOOST_AUTO_TEST_CASE(read_tar_file)
 
 BOOST_AUTO_TEST_CASE(write_tar_file)
 {
-    TemporaryFile tmp {TEST_DATA_DIR "/tar_write_test.tar"};
+    TemporaryFile tmp{TEST_DATA_DIR "/tar_write_test.tar"};
 
     std::uint64_t single_64bit_integer = 0xDEADBEEFAABBCCDD;
     std::uint32_t single_32bit_integer = 0xDEADBEEF;
@@ -62,8 +62,8 @@ BOOST_AUTO_TEST_CASE(write_tar_file)
 
     {
         storage::tar::FileWriter writer(tmp.path, storage::tar::FileWriter::GenerateFingerprint);
-        writer.WriteOne("foo/single_64bit_integer", single_64bit_integer);
+        writer.WriteFrom("foo/single_64bit_integer", single_64bit_integer);
-        writer.WriteOne("bar/single_32bit_integer", single_32bit_integer);
+        writer.WriteFrom("bar/single_32bit_integer", single_32bit_integer);
         writer.WriteElementCount64("baz/bla/64bit_vector", vector_64bit.size());
         writer.WriteFrom("baz/bla/64bit_vector", vector_64bit.data(), vector_64bit.size());
         writer.WriteElementCount64("32bit_vector", vector_32bit.size());
@@ -72,10 +72,12 @@ BOOST_AUTO_TEST_CASE(write_tar_file)
 
     storage::tar::FileReader reader(tmp.path, storage::tar::FileReader::VerifyFingerprint);
 
-    BOOST_CHECK_EQUAL(reader.ReadOne<std::uint32_t>("bar/single_32bit_integer"),
+    std::uint32_t result_32bit_integer;
-                      single_32bit_integer);
+    std::uint64_t result_64bit_integer;
-    BOOST_CHECK_EQUAL(reader.ReadOne<std::uint64_t>("foo/single_64bit_integer"),
+    reader.ReadInto("bar/single_32bit_integer", result_32bit_integer);
-                      single_64bit_integer);
+    reader.ReadInto("foo/single_64bit_integer", result_64bit_integer);
+    BOOST_CHECK_EQUAL(result_32bit_integer, single_32bit_integer);
+    BOOST_CHECK_EQUAL(result_32bit_integer, single_64bit_integer);
 
     std::vector<std::uint64_t> result_64bit_vector(
         reader.ReadElementCount64("baz/bla/64bit_vector"));

unit_tests/util/io.cpp

@@ -123,7 +123,7 @@ BOOST_AUTO_TEST_CASE(io_corrupt_fingerprint)
         osrm::storage::io::FileWriter outfile(IO_CORRUPT_FINGERPRINT_FILE,
                                               osrm::storage::io::FileWriter::HasNoFingerprint);
 
-        outfile.WriteOne(0xDEADBEEFCAFEFACE);
+        outfile.WriteFrom(0xDEADBEEFCAFEFACE);
         osrm::storage::serialization::write(outfile, v);
     }
 
@@ -154,7 +154,7 @@ BOOST_AUTO_TEST_CASE(io_incompatible_fingerprint)
                                                   osrm::storage::io::FileWriter::HasNoFingerprint);
 
             const auto fingerprint = osrm::util::FingerPrint::GetValid();
-            outfile.WriteOne(fingerprint);
+            outfile.WriteFrom(fingerprint);
             osrm::storage::serialization::write(outfile, v);
         }