Buffer turn data on disk to save memory

include/extractor/edge_based_graph_factory.hpp

@@ -159,9 +159,6 @@ class EdgeBasedGraphFactory
 
     NBGToEBG InsertEdgeBasedNode(const NodeID u, const NodeID v);
 
-    void FlushVectorToStream(storage::io::FileWriter &edge_data_file,
-                             std::vector<OriginalEdgeData> &original_edge_data_vector) const;
-
     std::size_t restricted_turns_counter;
     std::size_t skipped_uturns_counter;
     std::size_t skipped_barrier_turns_counter;

include/extractor/files.hpp

@@ -117,7 +117,8 @@ template <typename TurnDataT>
 inline void readTurnData(const boost::filesystem::path &path, TurnDataT &turn_data)
 {
     static_assert(std::is_same<TurnDataContainer, TurnDataT>::value ||
-                      std::is_same<TurnDataView, TurnDataT>::value,
+                      std::is_same<TurnDataView, TurnDataT>::value ||
+                      std::is_same<TurnDataExternalContainer, TurnDataT>::value,
                   "");
     const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
     storage::io::FileReader reader{path, fingerprint};
@@ -130,7 +131,8 @@ template <typename TurnDataT>
 inline void writeTurnData(const boost::filesystem::path &path, const TurnDataT &turn_data)
 {
     static_assert(std::is_same<TurnDataContainer, TurnDataT>::value ||
-                      std::is_same<TurnDataView, TurnDataT>::value,
+                      std::is_same<TurnDataView, TurnDataT>::value ||
+                      std::is_same<TurnDataExternalContainer, TurnDataT>::value,
                   "");
     const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
     storage::io::FileWriter writer{path, fingerprint};

include/extractor/turn_data_container.hpp

@@ -122,6 +122,7 @@ template <storage::Ownership Ownership> class TurnDataContainerImpl
 };
 }
 
+using TurnDataExternalContainer = detail::TurnDataContainerImpl<storage::Ownership::External>;
 using TurnDataContainer = detail::TurnDataContainerImpl<storage::Ownership::Container>;
 using TurnDataView = detail::TurnDataContainerImpl<storage::Ownership::View>;
 }

include/storage/serialization.hpp

@@ -2,15 +2,39 @@
 #define OSRM_STORAGE_SERIALIZATION_HPP
 
 #include "util/vector_view.hpp"
+#include "util/integer_range.hpp"
 
 #include "storage/io.hpp"
 
+#include <cstdint>
+
 namespace osrm
 {
 namespace storage
 {
 namespace serialization
 {
+template <typename T>
+inline void read(storage::io::FileReader &reader, stxxl::vector<T> &vec)
+{
+    auto size = reader.ReadOne<std::uint64_t>();
+    vec.reserve(size);
+    for (auto idx : util::irange<std::size_t>(0, size))
+    {
+        (void)idx;
+        vec.push_back(reader.ReadOne<T>());
+    }
+}
+
+template <typename T>
+inline void write(storage::io::FileWriter &writer, const stxxl::vector<T> &vec)
+{
+    writer.WriteOne(vec.size());
+    for (auto idx : util::irange<std::size_t>(0, vec.size()))
+    {
+        writer.WriteOne<T>(vec[idx]);
+    }
+}
 
 template <typename T> void read(io::FileReader &reader, std::vector<T> &data)
 {

include/storage/shared_memory_ownership.hpp

@@ -9,7 +9,8 @@ namespace storage
 enum class Ownership
 {
     Container,
-    View
+    View,
+    External
 };
 }
 }

include/util/vector_view.hpp

@@ -6,6 +6,8 @@
 
 #include "storage/shared_memory_ownership.hpp"
 
+#include <stxxl/vector>
+
 #include <boost/assert.hpp>
 #include <boost/iterator/iterator_facade.hpp>
 #include <boost/iterator/reverse_iterator.hpp>
@@ -174,10 +176,13 @@ template <typename DataT> void swap(vector_view<DataT> &lhs, vector_view<DataT>
     std::swap(lhs.m_size, rhs.m_size);
 }
 
+template <typename DataT, storage::Ownership Ownership>
+using InternalOrExternalVector = typename std::conditional<Ownership == storage::Ownership::External, stxxl::vector<DataT>, std::vector<DataT>>::type;
+
 template <typename DataT, storage::Ownership Ownership>
 using ViewOrVector = typename std::conditional<Ownership == storage::Ownership::View,
                                                vector_view<DataT>,
-                                               std::vector<DataT>>::type;
+                                               InternalOrExternalVector<DataT, Ownership>>::type;
 }
 }
 

src/extractor/edge_based_graph_factory.cpp

@@ -309,8 +309,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
     storage::io::FileWriter turn_penalties_index_file(turn_penalties_index_filename,
                                                       storage::io::FileWriter::HasNoFingerprint);
 
-    // TODO investigate increased peak memory consumption by keeping this in memory now
+    TurnDataExternalContainer turn_data_container;
-    TurnDataContainer turn_data_container;
 
     // Loop over all turns and generate new set of edges.
     // Three nested loop look super-linear, but we are dealing with a (kind of)