No shared_memory_wrapper, (De)SerializeVector

This commit is contained in:
Patrick Niklaus 2017-04-04 22:00:17 +00:00 committed by Patrick Niklaus
parent dce0ce0e17
commit d94017dfae
24 changed files with 292 additions and 321 deletions

View File

@ -2,7 +2,7 @@
#define OSRM_EXTRACTOR_FILES_HPP
#include "extractor/guidance/turn_lane_types.hpp"
#include "extractor/seralization.hpp"
#include "extractor/serialization.hpp"
#include "util/coordinate.hpp"
#include "util/packed_vector.hpp"
@ -20,26 +20,26 @@ namespace files
// reads .osrm.nodes
template<storage::Ownership Ownership>
inline void readNodes(const boost::filesystem::path &path,
typename util::ShM<util::Coordinate, Ownership>::vector &coordinates,
util::ViewOrVector<util::Coordinate, Ownership> &coordinates,
util::detail::PackedVector<OSMNodeID, Ownership> &osm_node_ids)
{
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(coordinates);
storage::serialization::read(reader, coordinates);
util::serialization::read(reader, osm_node_ids);
}
// writes .osrm.nodes
template<storage::Ownership Ownership>
inline void writeNodes(const boost::filesystem::path &path,
const typename util::ShM<util::Coordinate, Ownership>::vector &coordinates,
const util::ViewOrVector<util::Coordinate, Ownership> &coordinates,
const util::detail::PackedVector<OSMNodeID, Ownership> &osm_node_ids)
{
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(coordinates);
storage::serialization::write(writer, coordinates);
util::serialization::write(writer, osm_node_ids);
}
@ -49,7 +49,7 @@ inline void readNBGMapping(const boost::filesystem::path &path, std::vector<NBGT
const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint};
serialization::read(reader, mapping);
storage::serialization::read(reader, mapping);
}
// writes .osrm.cnbg_to_ebg
@ -59,7 +59,7 @@ inline void writeNBGMapping(const boost::filesystem::path &path,
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint};
serialization::write(writer, mapping);
storage::serialization::write(writer, mapping);
}
// reads .osrm.datasource_names
@ -127,27 +127,27 @@ inline void writeTurnData(const boost::filesystem::path &path,
// reads .osrm.tls
template <storage::Ownership Ownership>
inline void readTurnLaneDescriptions(const boost::filesystem::path &path,
typename util::ShM<std::uint32_t, Ownership>::vector &turn_offsets,
typename util::ShM<extractor::guidance::TurnLaneType::Mask, Ownership>::vector &turn_masks)
util::ViewOrVector<std::uint32_t, Ownership> &turn_offsets,
util::ViewOrVector<extractor::guidance::TurnLaneType::Mask, Ownership> &turn_masks)
{
const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(turn_offsets);
reader.DeserializeVector(turn_masks);
storage::serialization::read(reader, turn_offsets);
storage::serialization::read(reader, turn_masks);
}
// writes .osrm.tls
template <storage::Ownership Ownership>
inline void writeTurnLaneDescriptions(const boost::filesystem::path &path,
const typename util::ShM<std::uint32_t, Ownership>::vector &turn_offsets,
const typename util::ShM<extractor::guidance::TurnLaneType::Mask, Ownership>::vector &turn_masks)
const util::ViewOrVector<std::uint32_t, Ownership> &turn_offsets,
const util::ViewOrVector<extractor::guidance::TurnLaneType::Mask, Ownership> &turn_masks)
{
const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(turn_offsets);
writer.SerializeVector(turn_masks);
storage::serialization::write(writer, turn_offsets);
storage::serialization::write(writer, turn_masks);
}
}

View File

@ -1,7 +1,7 @@
#ifndef OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_
#define OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/vector_view.hpp"
#include "util/typedefs.hpp"
#include "storage/shared_memory_ownership.hpp"
@ -50,7 +50,7 @@ namespace detail
{
template <storage::Ownership Ownership> class SegmentDataContainerImpl
{
template <typename T> using Vector = typename util::ShM<T, Ownership>::vector;
template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
friend CompressedEdgeContainer;

View File

@ -1,97 +0,0 @@
#ifndef OSRM_EXTRACTOR_IO_HPP
#define OSRM_EXTRACTOR_IO_HPP
#include "extractor/datasources.hpp"
#include "extractor/nbg_to_ebg.hpp"
#include "extractor/segment_data_container.hpp"
#include "extractor/turn_data_container.hpp"
#include "storage/io.hpp"
#include <boost/assert.hpp>
namespace osrm
{
namespace extractor
{
namespace serialization
{
inline void read(storage::io::FileReader &reader, std::vector<NBGToEBG> &mapping)
{
reader.DeserializeVector(mapping);
}
inline void write(storage::io::FileWriter &writer, const std::vector<NBGToEBG> &mapping)
{
writer.SerializeVector(mapping);
}
inline void read(storage::io::FileReader &reader, Datasources &sources)
{
reader.ReadInto(sources);
}
inline void write(storage::io::FileWriter &writer, Datasources &sources)
{
writer.WriteFrom(sources);
}
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader,
detail::SegmentDataContainerImpl<Ownership> &segment_data)
{
reader.DeserializeVector(segment_data.index);
reader.DeserializeVector(segment_data.nodes);
reader.DeserializeVector(segment_data.fwd_weights);
reader.DeserializeVector(segment_data.rev_weights);
reader.DeserializeVector(segment_data.fwd_durations);
reader.DeserializeVector(segment_data.rev_durations);
reader.DeserializeVector(segment_data.datasources);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::SegmentDataContainerImpl<Ownership> &segment_data)
{
writer.SerializeVector(segment_data.index);
writer.SerializeVector(segment_data.nodes);
writer.SerializeVector(segment_data.fwd_weights);
writer.SerializeVector(segment_data.rev_weights);
writer.SerializeVector(segment_data.fwd_durations);
writer.SerializeVector(segment_data.rev_durations);
writer.SerializeVector(segment_data.datasources);
}
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader,
detail::TurnDataContainerImpl<Ownership> &turn_data_container)
{
reader.DeserializeVector(turn_data_container.geometry_ids);
reader.DeserializeVector(turn_data_container.name_ids);
reader.DeserializeVector(turn_data_container.turn_instructions);
reader.DeserializeVector(turn_data_container.lane_data_ids);
reader.DeserializeVector(turn_data_container.travel_modes);
reader.DeserializeVector(turn_data_container.entry_class_ids);
reader.DeserializeVector(turn_data_container.pre_turn_bearings);
reader.DeserializeVector(turn_data_container.post_turn_bearings);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::TurnDataContainerImpl<Ownership> &turn_data_container)
{
writer.SerializeVector(turn_data_container.geometry_ids);
writer.SerializeVector(turn_data_container.name_ids);
writer.SerializeVector(turn_data_container.turn_instructions);
writer.SerializeVector(turn_data_container.lane_data_ids);
writer.SerializeVector(turn_data_container.travel_modes);
writer.SerializeVector(turn_data_container.entry_class_ids);
writer.SerializeVector(turn_data_container.pre_turn_bearings);
writer.SerializeVector(turn_data_container.post_turn_bearings);
}
}
}
}
#endif

View File

@ -0,0 +1,88 @@
#ifndef OSRM_EXTRACTOR_IO_HPP
#define OSRM_EXTRACTOR_IO_HPP
#include "extractor/datasources.hpp"
#include "extractor/nbg_to_ebg.hpp"
#include "extractor/segment_data_container.hpp"
#include "extractor/turn_data_container.hpp"
#include "storage/io.hpp"
#include "storage/serialization.hpp"
#include <boost/assert.hpp>
namespace osrm
{
namespace extractor
{
namespace serialization
{
inline void read(storage::io::FileReader &reader, Datasources &sources)
{
reader.ReadInto(sources);
}
inline void write(storage::io::FileWriter &writer, Datasources &sources)
{
writer.WriteFrom(sources);
}
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader,
detail::SegmentDataContainerImpl<Ownership> &segment_data)
{
storage::serialization::read(reader, segment_data.index);
storage::serialization::read(reader, segment_data.nodes);
storage::serialization::read(reader, segment_data.fwd_weights);
storage::serialization::read(reader, segment_data.rev_weights);
storage::serialization::read(reader, segment_data.fwd_durations);
storage::serialization::read(reader, segment_data.rev_durations);
storage::serialization::read(reader, segment_data.datasources);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::SegmentDataContainerImpl<Ownership> &segment_data)
{
storage::serialization::write(writer, segment_data.index);
storage::serialization::write(writer, segment_data.nodes);
storage::serialization::write(writer, segment_data.fwd_weights);
storage::serialization::write(writer, segment_data.rev_weights);
storage::serialization::write(writer, segment_data.fwd_durations);
storage::serialization::write(writer, segment_data.rev_durations);
storage::serialization::write(writer, segment_data.datasources);
}
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader,
detail::TurnDataContainerImpl<Ownership> &turn_data_container)
{
storage::serialization::read(reader, turn_data_container.geometry_ids);
storage::serialization::read(reader, turn_data_container.name_ids);
storage::serialization::read(reader, turn_data_container.turn_instructions);
storage::serialization::read(reader, turn_data_container.lane_data_ids);
storage::serialization::read(reader, turn_data_container.travel_modes);
storage::serialization::read(reader, turn_data_container.entry_class_ids);
storage::serialization::read(reader, turn_data_container.pre_turn_bearings);
storage::serialization::read(reader, turn_data_container.post_turn_bearings);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::TurnDataContainerImpl<Ownership> &turn_data_container)
{
storage::serialization::write(writer, turn_data_container.geometry_ids);
storage::serialization::write(writer, turn_data_container.name_ids);
storage::serialization::write(writer, turn_data_container.turn_instructions);
storage::serialization::write(writer, turn_data_container.lane_data_ids);
storage::serialization::write(writer, turn_data_container.travel_modes);
storage::serialization::write(writer, turn_data_container.entry_class_ids);
storage::serialization::write(writer, turn_data_container.pre_turn_bearings);
storage::serialization::write(writer, turn_data_container.post_turn_bearings);
}
}
}
}
#endif

View File

@ -6,7 +6,7 @@
#include "storage/shared_memory_ownership.hpp"
#include "util/guidance/turn_bearing.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/vector_view.hpp"
#include "util/typedefs.hpp"
@ -43,7 +43,7 @@ namespace detail
{
template <storage::Ownership Ownership> class TurnDataContainerImpl
{
template <typename T> using Vector = typename util::ShM<T, Ownership>::vector;
template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
public:
TurnDataContainerImpl() = default;

View File

@ -6,7 +6,7 @@
#include "util/assert.hpp"
#include "util/for_each_range.hpp"
#include "util/log.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/vector_view.hpp"
#include "util/typedefs.hpp"
#include "storage/io.hpp"
@ -74,7 +74,7 @@ template <storage::Ownership Ownership> class CellStorageImpl
};
private:
template <typename T> using Vector = typename util::ShM<T, Ownership>::vector;
template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
// Implementation of the cell view. We need a template parameter here
// because we need to derive a read-only and read-write view from this.

View File

@ -6,6 +6,7 @@
#include "storage/shared_memory_ownership.hpp"
#include "util/static_graph.hpp"
#include "util/vector_view.hpp"
#include <tbb/parallel_sort.h>
@ -41,7 +42,7 @@ class MultiLevelGraph : public util::StaticGraph<EdgeDataT, Ownership>
{
private:
using SuperT = util::StaticGraph<EdgeDataT, Ownership>;
template <typename T> using Vector = typename util::ShM<T, Ownership>::vector;
template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
public:
// We limit each node to have 255 edges

View File

@ -4,7 +4,7 @@
#include "util/exception.hpp"
#include "util/for_each_pair.hpp"
#include "util/msb.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/vector_view.hpp"
#include "util/typedefs.hpp"
#include "storage/io.hpp"
@ -56,7 +56,7 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
static const constexpr std::uint8_t MAX_NUM_LEVEL = 16;
static const constexpr std::uint8_t NUM_PARTITION_BITS = sizeof(PartitionID) * CHAR_BIT;
template <typename T> using Vector = typename util::ShM<T, Ownership>::vector;
template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
public:
// Contains all data necessary to describe the level hierarchy

View File

@ -7,6 +7,7 @@
#include "partition/multi_level_partition.hpp"
#include "storage/io.hpp"
#include "storage/serialization.hpp"
#include "storage/shared_memory_ownership.hpp"
namespace osrm
@ -19,26 +20,26 @@ namespace serialization
template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, MultiLevelGraph<EdgeDataT, Ownership> &graph)
{
reader.DeserializeVector(graph.node_array);
reader.DeserializeVector(graph.edge_array);
reader.DeserializeVector(graph.node_to_edge_offset);
storage::serialization::read(reader, graph.node_array);
storage::serialization::read(reader, graph.edge_array);
storage::serialization::read(reader, graph.node_to_edge_offset);
}
template <typename EdgeDataT, storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const MultiLevelGraph<EdgeDataT, Ownership> &graph)
{
writer.SerializeVector(graph.node_array);
writer.SerializeVector(graph.edge_array);
writer.SerializeVector(graph.node_to_edge_offset);
storage::serialization::write(writer, graph.node_array);
storage::serialization::write(writer, graph.edge_array);
storage::serialization::write(writer, graph.node_to_edge_offset);
}
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
reader.ReadInto(*mlp.level_data);
reader.DeserializeVector(mlp.partition);
reader.DeserializeVector(mlp.cell_to_children);
storage::serialization::read(reader, mlp.partition);
storage::serialization::read(reader, mlp.cell_to_children);
}
template <storage::Ownership Ownership>
@ -46,29 +47,29 @@ inline void write(storage::io::FileWriter &writer,
const detail::MultiLevelPartitionImpl<Ownership> &mlp)
{
writer.WriteOne(*mlp.level_data);
writer.SerializeVector(mlp.partition);
writer.SerializeVector(mlp.cell_to_children);
storage::serialization::write(writer, mlp.partition);
storage::serialization::write(writer, mlp.cell_to_children);
}
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage)
{
reader.DeserializeVector(storage.weights);
reader.DeserializeVector(storage.source_boundary);
reader.DeserializeVector(storage.destination_boundary);
reader.DeserializeVector(storage.cells);
reader.DeserializeVector(storage.level_to_cell_offset);
storage::serialization::read(reader, storage.weights);
storage::serialization::read(reader, storage.source_boundary);
storage::serialization::read(reader, storage.destination_boundary);
storage::serialization::read(reader, storage.cells);
storage::serialization::read(reader, storage.level_to_cell_offset);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::CellStorageImpl<Ownership> &storage)
{
writer.SerializeVector(storage.weights);
writer.SerializeVector(storage.source_boundary);
writer.SerializeVector(storage.destination_boundary);
writer.SerializeVector(storage.cells);
writer.SerializeVector(storage.level_to_cell_offset);
storage::serialization::write(writer, storage.weights);
storage::serialization::write(writer, storage.source_boundary);
storage::serialization::write(writer, storage.destination_boundary);
storage::serialization::write(writer, storage.cells);
storage::serialization::write(writer, storage.level_to_cell_offset);
}
}
}

View File

@ -128,20 +128,6 @@ class FileReader
std::uint32_t ReadElementCount32() { return ReadOne<std::uint32_t>(); }
std::uint64_t ReadElementCount64() { return ReadOne<std::uint64_t>(); }
template <typename VectorT> void DeserializeVector(VectorT &data)
{
const auto count = ReadElementCount64();
data.resize(count);
ReadInto(data.data(), count);
}
template <typename T> std::size_t GetVectorMemorySize()
{
const auto count = ReadElementCount64();
Skip<T>(count);
return sizeof(count) + sizeof(T) * count;
}
template <typename T> std::size_t ReadVectorSize()
{
const auto count = ReadElementCount64();
@ -149,19 +135,6 @@ class FileReader
return count;
}
template <typename T> void *DeserializeVector(void *begin, const void *end)
{
auto count = ReadElementCount64();
auto required = reinterpret_cast<char *>(begin) + sizeof(count) + sizeof(T) * count;
if (required > end)
throw util::exception("Not enough memory ");
*reinterpret_cast<decltype(count) *>(begin) = count;
ReadInto(reinterpret_cast<T *>(reinterpret_cast<char *>(begin) + sizeof(decltype(count))),
count);
return required;
}
bool ReadAndCheckFingerprint()
{
auto loaded_fingerprint = ReadOne<util::FingerPrint>();
@ -199,42 +172,6 @@ class FileReader
input_stream.seekg(current_pos, input_stream.beg);
return length;
}
std::vector<std::string> ReadLines()
{
std::vector<std::string> result;
std::string thisline;
try
{
while (std::getline(input_stream, thisline))
{
result.push_back(thisline);
}
}
catch (const std::ios_base::failure &)
{
// EOF is OK here, everything else, re-throw
if (!input_stream.eof())
throw;
}
return result;
}
std::string ReadLine()
{
std::string thisline;
try
{
std::getline(input_stream, thisline);
}
catch (const std::ios_base::failure & /*e*/)
{
// EOF is OK here, everything else, re-throw
if (!input_stream.eof())
throw;
}
return thisline;
}
};
class FileWriter
@ -296,13 +233,6 @@ class FileWriter
void WriteElementCount32(const std::uint32_t count) { WriteOne<std::uint32_t>(count); }
void WriteElementCount64(const std::uint64_t count) { WriteOne<std::uint64_t>(count); }
template <typename VectorT> void SerializeVector(const VectorT &data)
{
const auto count = data.size();
WriteElementCount64(count);
return WriteFrom(data.data(), count);
}
void WriteFingerprint()
{
const auto fingerprint = util::FingerPrint::GetValid();

View File

@ -0,0 +1,47 @@
#ifndef OSRM_STORAGE_SERIALIZATION_HPP
#define OSRM_STORAGE_SERIALIZATION_HPP
#include "util/vector_view.hpp"
#include "storage/io.hpp"
namespace osrm
{
namespace storage
{
namespace serialization
{
template <typename T> void read(io::FileReader &reader, std::vector<T> &data)
{
const auto count = reader.ReadElementCount64();
data.resize(count);
reader.ReadInto(data.data(), count);
}
template <typename T> void write(io::FileWriter &writer, const std::vector<T> &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
return writer.WriteFrom(data.data(), count);
}
template <typename T> void read(io::FileReader &reader, util::vector_view<T> &data)
{
const auto count = reader.ReadElementCount64();
data.resize(count);
reader.ReadInto(data.data(), count);
}
template <typename T> void write(io::FileWriter &writer, const util::vector_view<T> &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
return writer.WriteFrom(data.data(), count);
}
}
}
}
#endif

View File

@ -2,7 +2,6 @@
#define OSRM_UTIL_NAME_TABLE_HPP
#include "util/indexed_data.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/string_view.hpp"
#include "util/typedefs.hpp"

View File

@ -1,7 +1,7 @@
#ifndef PACKED_VECTOR_HPP
#define PACKED_VECTOR_HPP
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/vector_view.hpp"
#include "util/typedefs.hpp"
#include "storage/io.hpp"
@ -171,7 +171,7 @@ template <typename T, storage::Ownership Ownership> class PackedVector
const detail::PackedVector<T, Ownership> &vec);
private:
typename util::ShM<std::uint64_t, Ownership>::vector vec;
util::ViewOrVector<std::uint64_t, Ownership> vec;
std::uint64_t num_elements = 0;

View File

@ -4,7 +4,7 @@
#include "storage/io.hpp"
#include "storage/shared_memory_ownership.hpp"
#include "util/integer_range.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/vector_view.hpp"
#include <array>
#include <fstream>
@ -41,8 +41,8 @@ template <unsigned BLOCK_SIZE, storage::Ownership Ownership> class RangeTable
{
public:
using BlockT = std::array<unsigned char, BLOCK_SIZE>;
using BlockContainerT = typename ShM<BlockT, Ownership>::vector;
using OffsetContainerT = typename ShM<unsigned, Ownership>::vector;
using BlockContainerT = util::ViewOrVector<BlockT, Ownership>;
using OffsetContainerT = util::ViewOrVector<unsigned, Ownership>;
using RangeT = range<unsigned>;
friend std::ostream &operator<<<>(std::ostream &out, const RangeTable &table);

View File

@ -4,7 +4,9 @@
#include "util/packed_vector.hpp"
#include "util/static_graph.hpp"
#include "util/dynamic_graph.hpp"
#include "storage/io.hpp"
#include "storage/serialization.hpp"
namespace osrm
{
@ -17,7 +19,7 @@ inline void read(storage::io::FileReader &reader,
detail::PackedVector<T, Ownership> &vec)
{
vec.num_elements =reader.ReadOne<std::uint64_t>();
reader.DeserializeVector(vec.vec);
storage::serialization::read(reader, vec.vec);
}
template <typename T, storage::Ownership Ownership>
@ -25,30 +27,30 @@ inline void write(storage::io::FileWriter &writer,
const detail::PackedVector<T, Ownership> &vec)
{
writer.WriteOne(vec.num_elements);
writer.SerializeVector(vec.vec);
storage::serialization::write(writer, vec.vec);
}
template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader,
StaticGraph<EdgeDataT, Ownership> &graph)
{
reader.DeserializeVector(graph.node_array);
reader.DeserializeVector(graph.edge_array);
storage::serialization::read(reader, graph.node_array);
storage::serialization::read(reader, graph.edge_array);
}
template <typename EdgeDataT, storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const StaticGraph<EdgeDataT, Ownership> &graph)
{
writer.SerializeVector(graph.node_array);
writer.SerializeVector(graph.edge_array);
storage::serialization::write(writer, graph.node_array);
storage::serialization::write(writer, graph.edge_array);
}
template <typename EdgeDataT>
inline void read(storage::io::FileReader &reader,
DynamicGraph<EdgeDataT> &graph)
{
reader.DeserializeVector(graph.node_array);
storage::serialization::read(reader, graph.node_array);
auto num_edges = reader.ReadElementCount64();
graph.edge_list.resize(num_edges);
for (auto index : irange<std::size_t>(0, num_edges))
@ -63,7 +65,7 @@ template <typename EdgeDataT>
inline void write(storage::io::FileWriter &writer,
const DynamicGraph<EdgeDataT> &graph)
{
writer.SerializeVector(graph.node_array);
storage::serialization::write(writer, graph.node_array);
writer.WriteElementCount64(graph.number_of_edges);
for (auto index : irange<std::size_t>(0, graph.number_of_edges))
{

View File

@ -4,7 +4,7 @@
#include "util/graph_traits.hpp"
#include "util/integer_range.hpp"
#include "util/percent.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/vector_view.hpp"
#include "util/typedefs.hpp"
#include "storage/shared_memory_ownership.hpp"
@ -122,7 +122,7 @@ EntryT edgeToEntry(const OtherEdge &from, std::false_type)
template <typename EdgeDataT, storage::Ownership Ownership = storage::Ownership::Container>
class StaticGraph
{
template <typename T> using Vector = typename util::ShM<T, Ownership>::vector;
template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
public:
using InputEdge = static_graph_details::SortableEdgeWithData<EdgeDataT>;

View File

@ -9,7 +9,7 @@
#include "util/hilbert_value.hpp"
#include "util/integer_range.hpp"
#include "util/rectangle.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/vector_view.hpp"
#include "util/typedefs.hpp"
#include "util/web_mercator.hpp"
@ -58,7 +58,7 @@ template <class EdgeDataT,
std::uint32_t LEAF_PAGE_SIZE = 4096>
class StaticRTree
{
template <typename T> using Vector = typename ShM<T, Ownership>::vector;
template <typename T> using Vector = ViewOrVector<T, Ownership>;
public:
using Rectangle = RectangleInt2D;
@ -160,7 +160,7 @@ class StaticRTree
boost::iostreams::mapped_file_source m_leaves_region;
// read-only view of leaves
typename ShM<const LeafNode, storage::Ownership::View>::vector m_leaves;
util::vector_view<const LeafNode> m_leaves;
public:
StaticRTree(const StaticRTree &) = delete;

View File

@ -1,5 +1,5 @@
#ifndef SHARED_MEMORY_VECTOR_WRAPPER_HPP
#define SHARED_MEMORY_VECTOR_WRAPPER_HPP
#ifndef UTIL_VECTOR_VIEW_HPP
#define UTIL_VECTOR_VIEW_HPP
#include "util/exception.hpp"
#include "util/log.hpp"
@ -25,11 +25,13 @@ namespace util
{
template <typename DataT>
class ShMemIterator
: public boost::iterator_facade<ShMemIterator<DataT>, DataT, boost::random_access_traversal_tag>
class VectorViewIterator : public boost::iterator_facade<VectorViewIterator<DataT>,
DataT,
boost::random_access_traversal_tag>
{
typedef boost::iterator_facade<ShMemIterator<DataT>, DataT, boost::random_access_traversal_tag>
base_t;
typedef boost::
iterator_facade<VectorViewIterator<DataT>, DataT, boost::random_access_traversal_tag>
base_t;
public:
typedef typename base_t::value_type value_type;
@ -37,16 +39,16 @@ class ShMemIterator
typedef typename base_t::reference reference;
typedef std::random_access_iterator_tag iterator_category;
explicit ShMemIterator() : m_value(nullptr) {}
explicit ShMemIterator(DataT *x) : m_value(x) {}
explicit VectorViewIterator() : m_value(nullptr) {}
explicit VectorViewIterator(DataT *x) : m_value(x) {}
private:
void increment() { ++m_value; }
void decrement() { --m_value; }
void advance(difference_type offset) { m_value += offset; }
bool equal(const ShMemIterator &other) const { return m_value == other.m_value; }
bool equal(const VectorViewIterator &other) const { return m_value == other.m_value; }
reference dereference() const { return *m_value; }
difference_type distance_to(const ShMemIterator &other) const
difference_type distance_to(const VectorViewIterator &other) const
{
return other.m_value - m_value;
}
@ -63,8 +65,8 @@ template <typename DataT> class vector_view
public:
using value_type = DataT;
using iterator = ShMemIterator<DataT>;
using const_iterator = ShMemIterator<const DataT>;
using iterator = VectorViewIterator<DataT>;
using const_iterator = VectorViewIterator<const DataT>;
using reverse_iterator = boost::reverse_iterator<iterator>;
vector_view() : m_ptr(nullptr), m_size(0) {}
@ -88,7 +90,9 @@ template <typename DataT> class vector_view
{
if (m_size != size)
{
throw util::exception("Invalid resize " + std::to_string(size) + " on immutable vector view of size " + std::to_string(m_size) + ".");
throw util::exception("Invalid resize " + std::to_string(size) +
" on immutable vector view of size " + std::to_string(m_size) +
".");
}
}
@ -189,12 +193,10 @@ template <typename DataT> void swap(vector_view<DataT> &lhs, vector_view<DataT>
std::swap(lhs.m_size, rhs.m_size);
}
template <typename DataT, storage::Ownership Ownership> struct ShM
{
using vector = typename std::conditional<Ownership == storage::Ownership::View,
vector_view<DataT>,
std::vector<DataT>>::type;
};
template <typename DataT, storage::Ownership Ownership>
using ViewOrVector = typename std::conditional<Ownership == storage::Ownership::View,
vector_view<DataT>,
std::vector<DataT>>::type;
}
}

View File

@ -1,6 +1,6 @@
#include "contractor/contractor.hpp"
#include "contractor/files.hpp"
#include "contractor/crc32_processor.hpp"
#include "contractor/files.hpp"
#include "contractor/graph_contractor.hpp"
#include "contractor/graph_contractor_adaptors.hpp"
@ -47,9 +47,9 @@ int Contractor::Run()
util::Log() << "Reading node weights.";
std::vector<EdgeWeight> node_weights;
{
storage::io::FileReader node_file(config.node_file_path,
storage::io::FileReader reader(config.node_file_path,
storage::io::FileReader::VerifyFingerprint);
node_file.DeserializeVector(node_weights);
storage::serialization::read(reader, node_weights);
}
util::Log() << "Done reading node weights.";
@ -115,10 +115,10 @@ void Contractor::WriteNodeLevels(std::vector<float> &&in_node_levels) const
{
std::vector<float> node_levels(std::move(in_node_levels));
storage::io::FileWriter node_level_file(config.level_output_path,
storage::io::FileWriter::HasNoFingerprint);
storage::io::FileWriter writer(config.level_output_path,
storage::io::FileWriter::HasNoFingerprint);
node_level_file.SerializeVector(node_levels);
storage::serialization::write(writer, node_levels);
}
void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const
@ -138,9 +138,8 @@ void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const
core_marker_output_file.WriteFrom(unpacked_bool_flags.data(), count);
}
void
Contractor::WriteContractedGraph(unsigned max_node_id,
util::DeallocatingVector<QueryEdge> contracted_edge_list)
void Contractor::WriteContractedGraph(unsigned max_node_id,
util::DeallocatingVector<QueryEdge> contracted_edge_list)
{
// Sorting contracted edges in a way that the static query graph can read some in in-place.
tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end());
@ -150,7 +149,7 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
RangebasedCRC32 crc32_calculator;
const unsigned checksum = crc32_calculator(contracted_edge_list);
QueryGraph query_graph {max_node_id+1, contracted_edge_list};
QueryGraph query_graph{max_node_id + 1, contracted_edge_list};
files::writeGraph(config.graph_output_path, checksum, query_graph);
}

View File

@ -48,8 +48,7 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory(
std::vector<std::uint32_t> &turn_lane_offsets,
std::vector<guidance::TurnLaneType::Mask> &turn_lane_masks,
guidance::LaneDescriptionMap &lane_description_map)
: m_max_edge_id(0), m_coordinates(coordinates),
m_osm_node_ids(osm_node_ids),
: m_max_edge_id(0), m_coordinates(coordinates), m_osm_node_ids(osm_node_ids),
m_node_based_graph(std::move(node_based_graph)),
m_restriction_map(std::move(restriction_map)), m_barrier_nodes(barrier_nodes),
m_traffic_lights(traffic_lights), m_compressed_edge_container(compressed_edge_container),
@ -540,16 +539,14 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
const bool isTrivial = m_compressed_edge_container.IsTrivial(incoming_edge);
const auto &from_node =
isTrivial
? m_osm_node_ids[node_along_road_entering]
: m_osm_node_ids[m_compressed_edge_container.GetLastEdgeSourceID(
incoming_edge)];
isTrivial ? m_osm_node_ids[node_along_road_entering]
: m_osm_node_ids[m_compressed_edge_container.GetLastEdgeSourceID(
incoming_edge)];
const auto &via_node =
m_osm_node_ids[m_compressed_edge_container.GetLastEdgeTargetID(
incoming_edge)];
const auto &to_node =
m_osm_node_ids[m_compressed_edge_container.GetFirstEdgeTargetID(
turn.eid)];
m_osm_node_ids[m_compressed_edge_container.GetFirstEdgeTargetID(turn.eid)];
lookup::TurnIndexBlock turn_index_block = {from_node, via_node, to_node};
@ -562,32 +559,32 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
// write weight penalties per turn
BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size());
{
storage::io::FileWriter turn_weight_penalties_file(
turn_weight_penalties_filename, storage::io::FileWriter::HasNoFingerprint);
turn_weight_penalties_file.SerializeVector(turn_weight_penalties);
storage::io::FileWriter writer(turn_weight_penalties_filename,
storage::io::FileWriter::HasNoFingerprint);
storage::serialization::write(writer, turn_weight_penalties);
}
{
storage::io::FileWriter turn_duration_penalties_file(
turn_duration_penalties_filename, storage::io::FileWriter::HasNoFingerprint);
turn_duration_penalties_file.SerializeVector(turn_duration_penalties);
storage::io::FileWriter writer(turn_duration_penalties_filename,
storage::io::FileWriter::HasNoFingerprint);
storage::serialization::write(writer, turn_duration_penalties);
}
util::Log() << "Created " << entry_class_hash.size() << " entry classes and "
<< bearing_class_hash.size() << " Bearing Classes";
util::Log() << "Writing Turn Lane Data to File...";
{
storage::io::FileWriter writer(turn_lane_data_filename,
storage::io::FileWriter::HasNoFingerprint);
storage::io::FileWriter turn_lane_data_file(turn_lane_data_filename,
storage::io::FileWriter::HasNoFingerprint);
std::vector<util::guidance::LaneTupleIdPair> lane_data(lane_data_map.size());
// extract lane data sorted by ID
for (auto itr : lane_data_map)
lane_data[itr.second] = itr.first;
turn_lane_data_file.SerializeVector(lane_data);
std::vector<util::guidance::LaneTupleIdPair> lane_data(lane_data_map.size());
// extract lane data sorted by ID
for (auto itr : lane_data_map)
lane_data[itr.second] = itr.first;
storage::serialization::write(writer, lane_data);
}
util::Log() << "done.";
files::writeTurnData(turn_data_filename, turn_data_container);

View File

@ -281,11 +281,11 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
util::Log() << "Saving edge-based node weights to file.";
TIMER_START(timer_write_node_weights);
storage::io::FileWriter edge_file(config.edge_based_node_weights_output_path,
storage::io::FileWriter::GenerateFingerprint);
edge_file.SerializeVector(edge_based_node_weights);
{
storage::io::FileWriter writer(config.edge_based_node_weights_output_path,
storage::io::FileWriter::GenerateFingerprint);
storage::serialization::write(writer, edge_based_node_weights);
}
TIMER_STOP(timer_write_node_weights);
util::Log() << "Done writing. (" << TIMER_SEC(timer_write_node_weights) << ")";
@ -299,7 +299,8 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
TIMER_STOP(rtree);
util::Log() << "Writing node map ...";
files::writeNodes<storage::Ownership::Container>(config.node_output_path, coordinates, osm_node_ids);
files::writeNodes<storage::Ownership::Container>(
config.node_output_path, coordinates, osm_node_ids);
WriteEdgeBasedGraph(config.edge_graph_output_path, max_edge_id, edge_based_edge_list);
@ -509,9 +510,8 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
};
compressed_node_based_graph_writing = std::async(std::launch::async, [&] {
WriteCompressedNodeBasedGraph(config.compressed_node_based_graph_output_path,
*node_based_graph,
coordinates);
WriteCompressedNodeBasedGraph(
config.compressed_node_based_graph_output_path, *node_based_graph, coordinates);
});
WriteTurnLaneData(config.turn_lane_descriptions_file_name);
@ -612,11 +612,11 @@ void Extractor::WriteIntersectionClassificationData(
const std::vector<util::guidance::BearingClass> &bearing_classes,
const std::vector<util::guidance::EntryClass> &entry_classes) const
{
storage::io::FileWriter file(output_file_name, storage::io::FileWriter::GenerateFingerprint);
storage::io::FileWriter writer(output_file_name, storage::io::FileWriter::GenerateFingerprint);
util::Log() << "Writing Intersection Classification Data";
TIMER_START(write_edges);
file.SerializeVector(node_based_intersection_classes);
storage::serialization::write(writer, node_based_intersection_classes);
// create range table for vectors:
std::vector<unsigned> bearing_counts;
@ -630,17 +630,17 @@ void Extractor::WriteIntersectionClassificationData(
}
util::RangeTable<> bearing_class_range_table(bearing_counts);
bearing_class_range_table.Write(file);
bearing_class_range_table.Write(writer);
file.WriteOne(total_bearings);
writer.WriteOne(total_bearings);
for (const auto &bearing_class : bearing_classes)
{
const auto &bearings = bearing_class.getAvailableBearings();
file.WriteFrom(bearings.data(), bearings.size());
writer.WriteFrom(bearings.data(), bearings.size());
}
file.SerializeVector(entry_classes);
storage::serialization::write(writer, entry_classes);
TIMER_STOP(write_edges);
util::Log() << "ok, after " << TIMER_SEC(write_edges) << "s for "
@ -659,9 +659,9 @@ void Extractor::WriteTurnLaneData(const std::string &turn_lane_file) const
util::Log() << "Writing turn lane masks...";
TIMER_START(turn_lane_timer);
storage::io::FileWriter file(turn_lane_file, storage::io::FileWriter::HasNoFingerprint);
file.SerializeVector(turn_lane_offsets);
file.SerializeVector(turn_lane_masks);
storage::io::FileWriter writer(turn_lane_file, storage::io::FileWriter::HasNoFingerprint);
storage::serialization::write(writer, turn_lane_offsets);
storage::serialization::write(writer, turn_lane_masks);
TIMER_STOP(turn_lane_timer);
util::Log() << "done (" << TIMER_SEC(turn_lane_timer) << ")";

View File

@ -34,10 +34,10 @@
#include "util/log.hpp"
#include "util/packed_vector.hpp"
#include "util/range_table.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/static_graph.hpp"
#include "util/static_rtree.hpp"
#include "util/typedefs.hpp"
#include "util/vector_view.hpp"
#ifdef __linux__
#include <sys/mman.h>
@ -63,7 +63,7 @@ namespace storage
{
using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf;
using RTreeNode = util:: StaticRTree<RTreeLeaf, storage::Ownership::View>::TreeNode;
using RTreeNode = util::StaticRTree<RTreeLeaf, storage::Ownership::View>::TreeNode;
using QueryGraph = util::StaticGraph<contractor::QueryEdge::EdgeData>;
using EdgeBasedGraph = util::StaticGraph<extractor::EdgeBasedEdge::EdgeData>;
@ -361,7 +361,7 @@ void Storage::PopulateLayout(DataLayout &layout)
io::FileReader::VerifyFingerprint);
std::vector<BearingClassID> bearing_class_id_table;
intersection_file.DeserializeVector(bearing_class_id_table);
serialization::read(intersection_file, bearing_class_id_table);
layout.SetBlockSize<BearingClassID>(DataLayout::BEARING_CLASSID,
bearing_class_id_table.size());
@ -386,7 +386,7 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlockSize<DiscreteBearing>(DataLayout::BEARING_VALUES, num_bearings);
std::vector<util::guidance::EntryClass> entry_class_table;
intersection_file.DeserializeVector(entry_class_table);
serialization::read(intersection_file, entry_class_table);
layout.SetBlockSize<util::guidance::EntryClass>(DataLayout::ENTRY_CLASS,
entry_class_table.size());
@ -690,11 +690,13 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
layout.GetBlockPtr<util::Coordinate, true>(memory_ptr, DataLayout::COORDINATE_LIST);
const auto osmnodeid_ptr =
layout.GetBlockPtr<std::uint64_t, true>(memory_ptr, DataLayout::OSM_NODE_ID_LIST);
util::vector_view<util::Coordinate> coordinates(coordinates_ptr, layout.num_entries[DataLayout::COORDINATE_LIST]);
util::vector_view<util::Coordinate> coordinates(
coordinates_ptr, layout.num_entries[DataLayout::COORDINATE_LIST]);
util::PackedVectorView<OSMNodeID> osm_node_ids;
osm_node_ids.reset(osmnodeid_ptr, layout.num_entries[DataLayout::OSM_NODE_ID_LIST]);
extractor::files::readNodes<storage::Ownership::View>(config.nodes_data_path, coordinates, osm_node_ids);
extractor::files::readNodes<storage::Ownership::View>(
config.nodes_data_path, coordinates, osm_node_ids);
}
// load turn weight penalties
@ -790,7 +792,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
io::FileReader::VerifyFingerprint);
std::vector<BearingClassID> bearing_class_id_table;
intersection_file.DeserializeVector(bearing_class_id_table);
serialization::read(intersection_file, bearing_class_id_table);
const auto bearing_blocks = intersection_file.ReadElementCount32();
intersection_file.Skip<std::uint32_t>(1); // sum_lengths
@ -808,7 +810,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
intersection_file.ReadInto(bearing_class_table.data(), num_bearings);
std::vector<util::guidance::EntryClass> entry_class_table;
intersection_file.DeserializeVector(entry_class_table);
serialization::read(intersection_file, entry_class_table);
// load intersection classes
if (!bearing_class_id_table.empty())

View File

@ -6,7 +6,6 @@
#include "extractor/edge_based_graph_factory.hpp"
#include "extractor/files.hpp"
#include "extractor/node_based_edge.hpp"
#include "extractor/seralization.hpp"
#include "storage/io.hpp"
@ -460,14 +459,14 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
const auto load_turn_weight_penalties = [&] {
using storage::io::FileReader;
FileReader file(config.turn_weight_penalties_path, FileReader::HasNoFingerprint);
file.DeserializeVector(turn_weight_penalties);
FileReader reader(config.turn_weight_penalties_path, FileReader::HasNoFingerprint);
storage::serialization::read(reader, turn_weight_penalties);
};
const auto load_turn_duration_penalties = [&] {
using storage::io::FileReader;
FileReader file(config.turn_duration_penalties_path, FileReader::HasNoFingerprint);
file.DeserializeVector(turn_duration_penalties);
FileReader reader(config.turn_duration_penalties_path, FileReader::HasNoFingerprint);
storage::serialization::read(reader, turn_duration_penalties);
};
const auto load_profile_properties = [&] {
@ -648,8 +647,8 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
if (update_turn_penalties)
{
const auto save_penalties = [](const auto &filename, const auto &data) -> void {
storage::io::FileWriter file(filename, storage::io::FileWriter::HasNoFingerprint);
file.SerializeVector(data);
storage::io::FileWriter writer(filename, storage::io::FileWriter::HasNoFingerprint);
storage::serialization::write(writer, data);
};
tbb::parallel_invoke(

View File

@ -1,4 +1,5 @@
#include "storage/io.hpp"
#include "storage/serialization.hpp"
#include "util/exception.hpp"
#include "util/fingerprint.hpp"
#include "util/typedefs.hpp"
@ -29,12 +30,12 @@ BOOST_AUTO_TEST_CASE(io_data)
{
osrm::storage::io::FileWriter outfile(IO_TMP_FILE,
osrm::storage::io::FileWriter::GenerateFingerprint);
outfile.SerializeVector(data_in);
osrm::storage::serialization::write(outfile, data_in);
}
osrm::storage::io::FileReader infile(IO_TMP_FILE,
osrm::storage::io::FileReader::VerifyFingerprint);
infile.DeserializeVector(data_out);
osrm::storage::serialization::read(infile, data_out);
BOOST_REQUIRE_EQUAL(data_in.size(), data_out.size());
BOOST_CHECK_EQUAL_COLLECTIONS(data_out.begin(), data_out.end(), data_in.begin(), data_in.end());
@ -65,7 +66,7 @@ BOOST_AUTO_TEST_CASE(file_too_small)
{
osrm::storage::io::FileWriter outfile(
IO_TOO_SMALL_FILE, osrm::storage::io::FileWriter::GenerateFingerprint);
outfile.SerializeVector(v);
osrm::storage::serialization::write(outfile, v);
}
std::fstream f(IO_TOO_SMALL_FILE);
@ -79,7 +80,7 @@ BOOST_AUTO_TEST_CASE(file_too_small)
osrm::storage::io::FileReader infile(IO_TOO_SMALL_FILE,
osrm::storage::io::FileReader::VerifyFingerprint);
std::vector<int> buffer;
infile.DeserializeVector(buffer);
osrm::storage::serialization::read(infile, buffer);
BOOST_REQUIRE_MESSAGE(false, "Should not get here");
}
catch (const osrm::util::exception &e)
@ -101,7 +102,7 @@ BOOST_AUTO_TEST_CASE(io_corrupt_fingerprint)
osrm::storage::io::FileWriter::HasNoFingerprint);
outfile.WriteOne(0xDEADBEEFCAFEFACE);
outfile.SerializeVector(v);
osrm::storage::serialization::write(outfile, v);
}
try
@ -130,7 +131,7 @@ BOOST_AUTO_TEST_CASE(io_incompatible_fingerprint)
const auto fingerprint = osrm::util::FingerPrint::GetValid();
outfile.WriteOne(fingerprint);
outfile.SerializeVector(v);
osrm::storage::serialization::write(outfile, v);
}
std::fstream f(IO_INCOMPATIBLE_FINGERPRINT_FILE);