Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Buffer turn data on disk to save memory

Browse Source
This commit is contained in:
Patrick Niklaus 2017-04-09 13:59:55 +00:00 committed by Patrick Niklaus
parent 4e9e2ed5bd
commit 59bbfeb67f
7 changed files with 38 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/extractor/edge_based_graph_factory.hpp
View File

@ -159,9 +159,6 @@ class EdgeBasedGraphFactory
NBGToEBG InsertEdgeBasedNode(const NodeID u, const NodeID v);
void FlushVectorToStream(storage::io::FileWriter &edge_data_file,
std::vector<OriginalEdgeData> &original_edge_data_vector) const;
std::size_t restricted_turns_counter;
std::size_t skipped_uturns_counter;
std::size_t skipped_barrier_turns_counter;

6
include/extractor/files.hpp
View File

@ -117,7 +117,8 @@ template <typename TurnDataT>
inline void readTurnData(const boost::filesystem::path &path, TurnDataT &turn_data)
{
static_assert(std::is_same<TurnDataContainer, TurnDataT>::value ||
std::is_same<TurnDataView, TurnDataT>::value,
std::is_same<TurnDataView, TurnDataT>::value ||
std::is_same<TurnDataExternalContainer, TurnDataT>::value,
"");
const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
storage::io::FileReader reader{path, fingerprint};
@ -130,7 +131,8 @@ template <typename TurnDataT>
inline void writeTurnData(const boost::filesystem::path &path, const TurnDataT &turn_data)
{
static_assert(std::is_same<TurnDataContainer, TurnDataT>::value ||
std::is_same<TurnDataView, TurnDataT>::value,
std::is_same<TurnDataView, TurnDataT>::value ||
std::is_same<TurnDataExternalContainer, TurnDataT>::value,
"");
const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
storage::io::FileWriter writer{path, fingerprint};

1
include/extractor/turn_data_container.hpp
View File

@ -122,6 +122,7 @@ template <storage::Ownership Ownership> class TurnDataContainerImpl
};
}
using TurnDataExternalContainer = detail::TurnDataContainerImpl<storage::Ownership::External>;
using TurnDataContainer = detail::TurnDataContainerImpl<storage::Ownership::Container>;
using TurnDataView = detail::TurnDataContainerImpl<storage::Ownership::View>;
}

24
include/storage/serialization.hpp
View File

@ -2,15 +2,39 @@
#define OSRM_STORAGE_SERIALIZATION_HPP
#include "util/vector_view.hpp"
#include "util/integer_range.hpp"
#include "storage/io.hpp"
#include <cstdint>
namespace osrm
{
namespace storage
{
namespace serialization
{
template <typename T>
inline void read(storage::io::FileReader &reader, stxxl::vector<T> &vec)
{
auto size = reader.ReadOne<std::uint64_t>();
vec.reserve(size);
for (auto idx : util::irange<std::size_t>(0, size))
{
(void)idx;
vec.push_back(reader.ReadOne<T>());
}
}
template <typename T>
inline void write(storage::io::FileWriter &writer, const stxxl::vector<T> &vec)
{
writer.WriteOne(vec.size());
for (auto idx : util::irange<std::size_t>(0, vec.size()))
{
writer.WriteOne<T>(vec[idx]);
}
}
template <typename T> void read(io::FileReader &reader, std::vector<T> &data)
{

3
include/storage/shared_memory_ownership.hpp
View File

@ -9,7 +9,8 @@ namespace storage
enum class Ownership
{
Container,
View
View,
External
};
}
}

7
include/util/vector_view.hpp
View File

@ -6,6 +6,8 @@
#include "storage/shared_memory_ownership.hpp"
#include <stxxl/vector>
#include <boost/assert.hpp>
#include <boost/iterator/iterator_facade.hpp>
#include <boost/iterator/reverse_iterator.hpp>
@ -174,10 +176,13 @@ template <typename DataT> void swap(vector_view<DataT> &lhs, vector_view<DataT>
std::swap(lhs.m_size, rhs.m_size);
}
template <typename DataT, storage::Ownership Ownership>
using InternalOrExternalVector = typename std::conditional<Ownership == storage::Ownership::External, stxxl::vector<DataT>, std::vector<DataT>>::type;
template <typename DataT, storage::Ownership Ownership>
using ViewOrVector = typename std::conditional<Ownership == storage::Ownership::View,
vector_view<DataT>,
std::vector<DataT>>::type;
InternalOrExternalVector<DataT, Ownership>>::type;
}
}

3
src/extractor/edge_based_graph_factory.cpp
View File

@ -309,8 +309,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
storage::io::FileWriter turn_penalties_index_file(turn_penalties_index_filename,
storage::io::FileWriter::HasNoFingerprint);
// TODO investigate increased peak memory consumption by keeping this in memory now
TurnDataContainer turn_data_container;
TurnDataExternalContainer turn_data_container;
// Loop over all turns and generate new set of edges.
// Three nested loop look super-linear, but we are dealing with a (kind of)