No shared_memory_wrapper, (De)SerializeVector

This commit is contained in:
Patrick Niklaus 2017-04-04 22:00:17 +00:00 committed by Patrick Niklaus
parent dce0ce0e17
commit d94017dfae
24 changed files with 292 additions and 321 deletions

View File

@ -2,7 +2,7 @@
#define OSRM_EXTRACTOR_FILES_HPP #define OSRM_EXTRACTOR_FILES_HPP
#include "extractor/guidance/turn_lane_types.hpp" #include "extractor/guidance/turn_lane_types.hpp"
#include "extractor/seralization.hpp" #include "extractor/serialization.hpp"
#include "util/coordinate.hpp" #include "util/coordinate.hpp"
#include "util/packed_vector.hpp" #include "util/packed_vector.hpp"
@ -20,26 +20,26 @@ namespace files
// reads .osrm.nodes // reads .osrm.nodes
template<storage::Ownership Ownership> template<storage::Ownership Ownership>
inline void readNodes(const boost::filesystem::path &path, inline void readNodes(const boost::filesystem::path &path,
typename util::ShM<util::Coordinate, Ownership>::vector &coordinates, util::ViewOrVector<util::Coordinate, Ownership> &coordinates,
util::detail::PackedVector<OSMNodeID, Ownership> &osm_node_ids) util::detail::PackedVector<OSMNodeID, Ownership> &osm_node_ids)
{ {
const auto fingerprint = storage::io::FileReader::VerifyFingerprint; const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint}; storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(coordinates); storage::serialization::read(reader, coordinates);
util::serialization::read(reader, osm_node_ids); util::serialization::read(reader, osm_node_ids);
} }
// writes .osrm.nodes // writes .osrm.nodes
template<storage::Ownership Ownership> template<storage::Ownership Ownership>
inline void writeNodes(const boost::filesystem::path &path, inline void writeNodes(const boost::filesystem::path &path,
const typename util::ShM<util::Coordinate, Ownership>::vector &coordinates, const util::ViewOrVector<util::Coordinate, Ownership> &coordinates,
const util::detail::PackedVector<OSMNodeID, Ownership> &osm_node_ids) const util::detail::PackedVector<OSMNodeID, Ownership> &osm_node_ids)
{ {
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint}; storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(coordinates); storage::serialization::write(writer, coordinates);
util::serialization::write(writer, osm_node_ids); util::serialization::write(writer, osm_node_ids);
} }
@ -49,7 +49,7 @@ inline void readNBGMapping(const boost::filesystem::path &path, std::vector<NBGT
const auto fingerprint = storage::io::FileReader::VerifyFingerprint; const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
storage::io::FileReader reader{path, fingerprint}; storage::io::FileReader reader{path, fingerprint};
serialization::read(reader, mapping); storage::serialization::read(reader, mapping);
} }
// writes .osrm.cnbg_to_ebg // writes .osrm.cnbg_to_ebg
@ -59,7 +59,7 @@ inline void writeNBGMapping(const boost::filesystem::path &path,
const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
storage::io::FileWriter writer{path, fingerprint}; storage::io::FileWriter writer{path, fingerprint};
serialization::write(writer, mapping); storage::serialization::write(writer, mapping);
} }
// reads .osrm.datasource_names // reads .osrm.datasource_names
@ -127,27 +127,27 @@ inline void writeTurnData(const boost::filesystem::path &path,
// reads .osrm.tls // reads .osrm.tls
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void readTurnLaneDescriptions(const boost::filesystem::path &path, inline void readTurnLaneDescriptions(const boost::filesystem::path &path,
typename util::ShM<std::uint32_t, Ownership>::vector &turn_offsets, util::ViewOrVector<std::uint32_t, Ownership> &turn_offsets,
typename util::ShM<extractor::guidance::TurnLaneType::Mask, Ownership>::vector &turn_masks) util::ViewOrVector<extractor::guidance::TurnLaneType::Mask, Ownership> &turn_masks)
{ {
const auto fingerprint = storage::io::FileReader::HasNoFingerprint; const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
storage::io::FileReader reader{path, fingerprint}; storage::io::FileReader reader{path, fingerprint};
reader.DeserializeVector(turn_offsets); storage::serialization::read(reader, turn_offsets);
reader.DeserializeVector(turn_masks); storage::serialization::read(reader, turn_masks);
} }
// writes .osrm.tls // writes .osrm.tls
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void writeTurnLaneDescriptions(const boost::filesystem::path &path, inline void writeTurnLaneDescriptions(const boost::filesystem::path &path,
const typename util::ShM<std::uint32_t, Ownership>::vector &turn_offsets, const util::ViewOrVector<std::uint32_t, Ownership> &turn_offsets,
const typename util::ShM<extractor::guidance::TurnLaneType::Mask, Ownership>::vector &turn_masks) const util::ViewOrVector<extractor::guidance::TurnLaneType::Mask, Ownership> &turn_masks)
{ {
const auto fingerprint = storage::io::FileWriter::HasNoFingerprint; const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
storage::io::FileWriter writer{path, fingerprint}; storage::io::FileWriter writer{path, fingerprint};
writer.SerializeVector(turn_offsets); storage::serialization::write(writer, turn_offsets);
writer.SerializeVector(turn_masks); storage::serialization::write(writer, turn_masks);
} }
} }

View File

@ -1,7 +1,7 @@
#ifndef OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_ #ifndef OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_
#define OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_ #define OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_
#include "util/shared_memory_vector_wrapper.hpp" #include "util/vector_view.hpp"
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
#include "storage/shared_memory_ownership.hpp" #include "storage/shared_memory_ownership.hpp"
@ -50,7 +50,7 @@ namespace detail
{ {
template <storage::Ownership Ownership> class SegmentDataContainerImpl template <storage::Ownership Ownership> class SegmentDataContainerImpl
{ {
template <typename T> using Vector = typename util::ShM<T, Ownership>::vector; template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
friend CompressedEdgeContainer; friend CompressedEdgeContainer;

View File

@ -1,97 +0,0 @@
#ifndef OSRM_EXTRACTOR_IO_HPP
#define OSRM_EXTRACTOR_IO_HPP
#include "extractor/datasources.hpp"
#include "extractor/nbg_to_ebg.hpp"
#include "extractor/segment_data_container.hpp"
#include "extractor/turn_data_container.hpp"
#include "storage/io.hpp"
#include <boost/assert.hpp>
namespace osrm
{
namespace extractor
{
namespace serialization
{
inline void read(storage::io::FileReader &reader, std::vector<NBGToEBG> &mapping)
{
reader.DeserializeVector(mapping);
}
inline void write(storage::io::FileWriter &writer, const std::vector<NBGToEBG> &mapping)
{
writer.SerializeVector(mapping);
}
inline void read(storage::io::FileReader &reader, Datasources &sources)
{
reader.ReadInto(sources);
}
inline void write(storage::io::FileWriter &writer, Datasources &sources)
{
writer.WriteFrom(sources);
}
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader,
detail::SegmentDataContainerImpl<Ownership> &segment_data)
{
reader.DeserializeVector(segment_data.index);
reader.DeserializeVector(segment_data.nodes);
reader.DeserializeVector(segment_data.fwd_weights);
reader.DeserializeVector(segment_data.rev_weights);
reader.DeserializeVector(segment_data.fwd_durations);
reader.DeserializeVector(segment_data.rev_durations);
reader.DeserializeVector(segment_data.datasources);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::SegmentDataContainerImpl<Ownership> &segment_data)
{
writer.SerializeVector(segment_data.index);
writer.SerializeVector(segment_data.nodes);
writer.SerializeVector(segment_data.fwd_weights);
writer.SerializeVector(segment_data.rev_weights);
writer.SerializeVector(segment_data.fwd_durations);
writer.SerializeVector(segment_data.rev_durations);
writer.SerializeVector(segment_data.datasources);
}
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader,
detail::TurnDataContainerImpl<Ownership> &turn_data_container)
{
reader.DeserializeVector(turn_data_container.geometry_ids);
reader.DeserializeVector(turn_data_container.name_ids);
reader.DeserializeVector(turn_data_container.turn_instructions);
reader.DeserializeVector(turn_data_container.lane_data_ids);
reader.DeserializeVector(turn_data_container.travel_modes);
reader.DeserializeVector(turn_data_container.entry_class_ids);
reader.DeserializeVector(turn_data_container.pre_turn_bearings);
reader.DeserializeVector(turn_data_container.post_turn_bearings);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::TurnDataContainerImpl<Ownership> &turn_data_container)
{
writer.SerializeVector(turn_data_container.geometry_ids);
writer.SerializeVector(turn_data_container.name_ids);
writer.SerializeVector(turn_data_container.turn_instructions);
writer.SerializeVector(turn_data_container.lane_data_ids);
writer.SerializeVector(turn_data_container.travel_modes);
writer.SerializeVector(turn_data_container.entry_class_ids);
writer.SerializeVector(turn_data_container.pre_turn_bearings);
writer.SerializeVector(turn_data_container.post_turn_bearings);
}
}
}
}
#endif

View File

@ -0,0 +1,88 @@
#ifndef OSRM_EXTRACTOR_IO_HPP
#define OSRM_EXTRACTOR_IO_HPP
#include "extractor/datasources.hpp"
#include "extractor/nbg_to_ebg.hpp"
#include "extractor/segment_data_container.hpp"
#include "extractor/turn_data_container.hpp"
#include "storage/io.hpp"
#include "storage/serialization.hpp"
#include <boost/assert.hpp>
namespace osrm
{
namespace extractor
{
namespace serialization
{
inline void read(storage::io::FileReader &reader, Datasources &sources)
{
reader.ReadInto(sources);
}
inline void write(storage::io::FileWriter &writer, Datasources &sources)
{
writer.WriteFrom(sources);
}
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader,
detail::SegmentDataContainerImpl<Ownership> &segment_data)
{
storage::serialization::read(reader, segment_data.index);
storage::serialization::read(reader, segment_data.nodes);
storage::serialization::read(reader, segment_data.fwd_weights);
storage::serialization::read(reader, segment_data.rev_weights);
storage::serialization::read(reader, segment_data.fwd_durations);
storage::serialization::read(reader, segment_data.rev_durations);
storage::serialization::read(reader, segment_data.datasources);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::SegmentDataContainerImpl<Ownership> &segment_data)
{
storage::serialization::write(writer, segment_data.index);
storage::serialization::write(writer, segment_data.nodes);
storage::serialization::write(writer, segment_data.fwd_weights);
storage::serialization::write(writer, segment_data.rev_weights);
storage::serialization::write(writer, segment_data.fwd_durations);
storage::serialization::write(writer, segment_data.rev_durations);
storage::serialization::write(writer, segment_data.datasources);
}
template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader,
detail::TurnDataContainerImpl<Ownership> &turn_data_container)
{
storage::serialization::read(reader, turn_data_container.geometry_ids);
storage::serialization::read(reader, turn_data_container.name_ids);
storage::serialization::read(reader, turn_data_container.turn_instructions);
storage::serialization::read(reader, turn_data_container.lane_data_ids);
storage::serialization::read(reader, turn_data_container.travel_modes);
storage::serialization::read(reader, turn_data_container.entry_class_ids);
storage::serialization::read(reader, turn_data_container.pre_turn_bearings);
storage::serialization::read(reader, turn_data_container.post_turn_bearings);
}
template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer,
const detail::TurnDataContainerImpl<Ownership> &turn_data_container)
{
storage::serialization::write(writer, turn_data_container.geometry_ids);
storage::serialization::write(writer, turn_data_container.name_ids);
storage::serialization::write(writer, turn_data_container.turn_instructions);
storage::serialization::write(writer, turn_data_container.lane_data_ids);
storage::serialization::write(writer, turn_data_container.travel_modes);
storage::serialization::write(writer, turn_data_container.entry_class_ids);
storage::serialization::write(writer, turn_data_container.pre_turn_bearings);
storage::serialization::write(writer, turn_data_container.post_turn_bearings);
}
}
}
}
#endif

View File

@ -6,7 +6,7 @@
#include "storage/shared_memory_ownership.hpp" #include "storage/shared_memory_ownership.hpp"
#include "util/guidance/turn_bearing.hpp" #include "util/guidance/turn_bearing.hpp"
#include "util/shared_memory_vector_wrapper.hpp" #include "util/vector_view.hpp"
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
@ -43,7 +43,7 @@ namespace detail
{ {
template <storage::Ownership Ownership> class TurnDataContainerImpl template <storage::Ownership Ownership> class TurnDataContainerImpl
{ {
template <typename T> using Vector = typename util::ShM<T, Ownership>::vector; template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
public: public:
TurnDataContainerImpl() = default; TurnDataContainerImpl() = default;

View File

@ -6,7 +6,7 @@
#include "util/assert.hpp" #include "util/assert.hpp"
#include "util/for_each_range.hpp" #include "util/for_each_range.hpp"
#include "util/log.hpp" #include "util/log.hpp"
#include "util/shared_memory_vector_wrapper.hpp" #include "util/vector_view.hpp"
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
#include "storage/io.hpp" #include "storage/io.hpp"
@ -74,7 +74,7 @@ template <storage::Ownership Ownership> class CellStorageImpl
}; };
private: private:
template <typename T> using Vector = typename util::ShM<T, Ownership>::vector; template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
// Implementation of the cell view. We need a template parameter here // Implementation of the cell view. We need a template parameter here
// because we need to derive a read-only and read-write view from this. // because we need to derive a read-only and read-write view from this.

View File

@ -6,6 +6,7 @@
#include "storage/shared_memory_ownership.hpp" #include "storage/shared_memory_ownership.hpp"
#include "util/static_graph.hpp" #include "util/static_graph.hpp"
#include "util/vector_view.hpp"
#include <tbb/parallel_sort.h> #include <tbb/parallel_sort.h>
@ -41,7 +42,7 @@ class MultiLevelGraph : public util::StaticGraph<EdgeDataT, Ownership>
{ {
private: private:
using SuperT = util::StaticGraph<EdgeDataT, Ownership>; using SuperT = util::StaticGraph<EdgeDataT, Ownership>;
template <typename T> using Vector = typename util::ShM<T, Ownership>::vector; template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
public: public:
// We limit each node to have 255 edges // We limit each node to have 255 edges

View File

@ -4,7 +4,7 @@
#include "util/exception.hpp" #include "util/exception.hpp"
#include "util/for_each_pair.hpp" #include "util/for_each_pair.hpp"
#include "util/msb.hpp" #include "util/msb.hpp"
#include "util/shared_memory_vector_wrapper.hpp" #include "util/vector_view.hpp"
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
#include "storage/io.hpp" #include "storage/io.hpp"
@ -56,7 +56,7 @@ template <storage::Ownership Ownership> class MultiLevelPartitionImpl final
static const constexpr std::uint8_t MAX_NUM_LEVEL = 16; static const constexpr std::uint8_t MAX_NUM_LEVEL = 16;
static const constexpr std::uint8_t NUM_PARTITION_BITS = sizeof(PartitionID) * CHAR_BIT; static const constexpr std::uint8_t NUM_PARTITION_BITS = sizeof(PartitionID) * CHAR_BIT;
template <typename T> using Vector = typename util::ShM<T, Ownership>::vector; template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
public: public:
// Contains all data necessary to describe the level hierarchy // Contains all data necessary to describe the level hierarchy

View File

@ -7,6 +7,7 @@
#include "partition/multi_level_partition.hpp" #include "partition/multi_level_partition.hpp"
#include "storage/io.hpp" #include "storage/io.hpp"
#include "storage/serialization.hpp"
#include "storage/shared_memory_ownership.hpp" #include "storage/shared_memory_ownership.hpp"
namespace osrm namespace osrm
@ -19,26 +20,26 @@ namespace serialization
template <typename EdgeDataT, storage::Ownership Ownership> template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, MultiLevelGraph<EdgeDataT, Ownership> &graph) inline void read(storage::io::FileReader &reader, MultiLevelGraph<EdgeDataT, Ownership> &graph)
{ {
reader.DeserializeVector(graph.node_array); storage::serialization::read(reader, graph.node_array);
reader.DeserializeVector(graph.edge_array); storage::serialization::read(reader, graph.edge_array);
reader.DeserializeVector(graph.node_to_edge_offset); storage::serialization::read(reader, graph.node_to_edge_offset);
} }
template <typename EdgeDataT, storage::Ownership Ownership> template <typename EdgeDataT, storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, inline void write(storage::io::FileWriter &writer,
const MultiLevelGraph<EdgeDataT, Ownership> &graph) const MultiLevelGraph<EdgeDataT, Ownership> &graph)
{ {
writer.SerializeVector(graph.node_array); storage::serialization::write(writer, graph.node_array);
writer.SerializeVector(graph.edge_array); storage::serialization::write(writer, graph.edge_array);
writer.SerializeVector(graph.node_to_edge_offset); storage::serialization::write(writer, graph.node_to_edge_offset);
} }
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp) inline void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl<Ownership> &mlp)
{ {
reader.ReadInto(*mlp.level_data); reader.ReadInto(*mlp.level_data);
reader.DeserializeVector(mlp.partition); storage::serialization::read(reader, mlp.partition);
reader.DeserializeVector(mlp.cell_to_children); storage::serialization::read(reader, mlp.cell_to_children);
} }
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
@ -46,29 +47,29 @@ inline void write(storage::io::FileWriter &writer,
const detail::MultiLevelPartitionImpl<Ownership> &mlp) const detail::MultiLevelPartitionImpl<Ownership> &mlp)
{ {
writer.WriteOne(*mlp.level_data); writer.WriteOne(*mlp.level_data);
writer.SerializeVector(mlp.partition); storage::serialization::write(writer, mlp.partition);
writer.SerializeVector(mlp.cell_to_children); storage::serialization::write(writer, mlp.cell_to_children);
} }
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage) inline void read(storage::io::FileReader &reader, detail::CellStorageImpl<Ownership> &storage)
{ {
reader.DeserializeVector(storage.weights); storage::serialization::read(reader, storage.weights);
reader.DeserializeVector(storage.source_boundary); storage::serialization::read(reader, storage.source_boundary);
reader.DeserializeVector(storage.destination_boundary); storage::serialization::read(reader, storage.destination_boundary);
reader.DeserializeVector(storage.cells); storage::serialization::read(reader, storage.cells);
reader.DeserializeVector(storage.level_to_cell_offset); storage::serialization::read(reader, storage.level_to_cell_offset);
} }
template <storage::Ownership Ownership> template <storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, inline void write(storage::io::FileWriter &writer,
const detail::CellStorageImpl<Ownership> &storage) const detail::CellStorageImpl<Ownership> &storage)
{ {
writer.SerializeVector(storage.weights); storage::serialization::write(writer, storage.weights);
writer.SerializeVector(storage.source_boundary); storage::serialization::write(writer, storage.source_boundary);
writer.SerializeVector(storage.destination_boundary); storage::serialization::write(writer, storage.destination_boundary);
writer.SerializeVector(storage.cells); storage::serialization::write(writer, storage.cells);
writer.SerializeVector(storage.level_to_cell_offset); storage::serialization::write(writer, storage.level_to_cell_offset);
} }
} }
} }

View File

@ -128,20 +128,6 @@ class FileReader
std::uint32_t ReadElementCount32() { return ReadOne<std::uint32_t>(); } std::uint32_t ReadElementCount32() { return ReadOne<std::uint32_t>(); }
std::uint64_t ReadElementCount64() { return ReadOne<std::uint64_t>(); } std::uint64_t ReadElementCount64() { return ReadOne<std::uint64_t>(); }
template <typename VectorT> void DeserializeVector(VectorT &data)
{
const auto count = ReadElementCount64();
data.resize(count);
ReadInto(data.data(), count);
}
template <typename T> std::size_t GetVectorMemorySize()
{
const auto count = ReadElementCount64();
Skip<T>(count);
return sizeof(count) + sizeof(T) * count;
}
template <typename T> std::size_t ReadVectorSize() template <typename T> std::size_t ReadVectorSize()
{ {
const auto count = ReadElementCount64(); const auto count = ReadElementCount64();
@ -149,19 +135,6 @@ class FileReader
return count; return count;
} }
template <typename T> void *DeserializeVector(void *begin, const void *end)
{
auto count = ReadElementCount64();
auto required = reinterpret_cast<char *>(begin) + sizeof(count) + sizeof(T) * count;
if (required > end)
throw util::exception("Not enough memory ");
*reinterpret_cast<decltype(count) *>(begin) = count;
ReadInto(reinterpret_cast<T *>(reinterpret_cast<char *>(begin) + sizeof(decltype(count))),
count);
return required;
}
bool ReadAndCheckFingerprint() bool ReadAndCheckFingerprint()
{ {
auto loaded_fingerprint = ReadOne<util::FingerPrint>(); auto loaded_fingerprint = ReadOne<util::FingerPrint>();
@ -199,42 +172,6 @@ class FileReader
input_stream.seekg(current_pos, input_stream.beg); input_stream.seekg(current_pos, input_stream.beg);
return length; return length;
} }
std::vector<std::string> ReadLines()
{
std::vector<std::string> result;
std::string thisline;
try
{
while (std::getline(input_stream, thisline))
{
result.push_back(thisline);
}
}
catch (const std::ios_base::failure &)
{
// EOF is OK here, everything else, re-throw
if (!input_stream.eof())
throw;
}
return result;
}
std::string ReadLine()
{
std::string thisline;
try
{
std::getline(input_stream, thisline);
}
catch (const std::ios_base::failure & /*e*/)
{
// EOF is OK here, everything else, re-throw
if (!input_stream.eof())
throw;
}
return thisline;
}
}; };
class FileWriter class FileWriter
@ -296,13 +233,6 @@ class FileWriter
void WriteElementCount32(const std::uint32_t count) { WriteOne<std::uint32_t>(count); } void WriteElementCount32(const std::uint32_t count) { WriteOne<std::uint32_t>(count); }
void WriteElementCount64(const std::uint64_t count) { WriteOne<std::uint64_t>(count); } void WriteElementCount64(const std::uint64_t count) { WriteOne<std::uint64_t>(count); }
template <typename VectorT> void SerializeVector(const VectorT &data)
{
const auto count = data.size();
WriteElementCount64(count);
return WriteFrom(data.data(), count);
}
void WriteFingerprint() void WriteFingerprint()
{ {
const auto fingerprint = util::FingerPrint::GetValid(); const auto fingerprint = util::FingerPrint::GetValid();

View File

@ -0,0 +1,47 @@
#ifndef OSRM_STORAGE_SERIALIZATION_HPP
#define OSRM_STORAGE_SERIALIZATION_HPP
#include "util/vector_view.hpp"
#include "storage/io.hpp"
namespace osrm
{
namespace storage
{
namespace serialization
{
template <typename T> void read(io::FileReader &reader, std::vector<T> &data)
{
const auto count = reader.ReadElementCount64();
data.resize(count);
reader.ReadInto(data.data(), count);
}
template <typename T> void write(io::FileWriter &writer, const std::vector<T> &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
return writer.WriteFrom(data.data(), count);
}
template <typename T> void read(io::FileReader &reader, util::vector_view<T> &data)
{
const auto count = reader.ReadElementCount64();
data.resize(count);
reader.ReadInto(data.data(), count);
}
template <typename T> void write(io::FileWriter &writer, const util::vector_view<T> &data)
{
const auto count = data.size();
writer.WriteElementCount64(count);
return writer.WriteFrom(data.data(), count);
}
}
}
}
#endif

View File

@ -2,7 +2,6 @@
#define OSRM_UTIL_NAME_TABLE_HPP #define OSRM_UTIL_NAME_TABLE_HPP
#include "util/indexed_data.hpp" #include "util/indexed_data.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/string_view.hpp" #include "util/string_view.hpp"
#include "util/typedefs.hpp" #include "util/typedefs.hpp"

View File

@ -1,7 +1,7 @@
#ifndef PACKED_VECTOR_HPP #ifndef PACKED_VECTOR_HPP
#define PACKED_VECTOR_HPP #define PACKED_VECTOR_HPP
#include "util/shared_memory_vector_wrapper.hpp" #include "util/vector_view.hpp"
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
#include "storage/io.hpp" #include "storage/io.hpp"
@ -171,7 +171,7 @@ template <typename T, storage::Ownership Ownership> class PackedVector
const detail::PackedVector<T, Ownership> &vec); const detail::PackedVector<T, Ownership> &vec);
private: private:
typename util::ShM<std::uint64_t, Ownership>::vector vec; util::ViewOrVector<std::uint64_t, Ownership> vec;
std::uint64_t num_elements = 0; std::uint64_t num_elements = 0;

View File

@ -4,7 +4,7 @@
#include "storage/io.hpp" #include "storage/io.hpp"
#include "storage/shared_memory_ownership.hpp" #include "storage/shared_memory_ownership.hpp"
#include "util/integer_range.hpp" #include "util/integer_range.hpp"
#include "util/shared_memory_vector_wrapper.hpp" #include "util/vector_view.hpp"
#include <array> #include <array>
#include <fstream> #include <fstream>
@ -41,8 +41,8 @@ template <unsigned BLOCK_SIZE, storage::Ownership Ownership> class RangeTable
{ {
public: public:
using BlockT = std::array<unsigned char, BLOCK_SIZE>; using BlockT = std::array<unsigned char, BLOCK_SIZE>;
using BlockContainerT = typename ShM<BlockT, Ownership>::vector; using BlockContainerT = util::ViewOrVector<BlockT, Ownership>;
using OffsetContainerT = typename ShM<unsigned, Ownership>::vector; using OffsetContainerT = util::ViewOrVector<unsigned, Ownership>;
using RangeT = range<unsigned>; using RangeT = range<unsigned>;
friend std::ostream &operator<<<>(std::ostream &out, const RangeTable &table); friend std::ostream &operator<<<>(std::ostream &out, const RangeTable &table);

View File

@ -4,7 +4,9 @@
#include "util/packed_vector.hpp" #include "util/packed_vector.hpp"
#include "util/static_graph.hpp" #include "util/static_graph.hpp"
#include "util/dynamic_graph.hpp" #include "util/dynamic_graph.hpp"
#include "storage/io.hpp" #include "storage/io.hpp"
#include "storage/serialization.hpp"
namespace osrm namespace osrm
{ {
@ -17,7 +19,7 @@ inline void read(storage::io::FileReader &reader,
detail::PackedVector<T, Ownership> &vec) detail::PackedVector<T, Ownership> &vec)
{ {
vec.num_elements =reader.ReadOne<std::uint64_t>(); vec.num_elements =reader.ReadOne<std::uint64_t>();
reader.DeserializeVector(vec.vec); storage::serialization::read(reader, vec.vec);
} }
template <typename T, storage::Ownership Ownership> template <typename T, storage::Ownership Ownership>
@ -25,30 +27,30 @@ inline void write(storage::io::FileWriter &writer,
const detail::PackedVector<T, Ownership> &vec) const detail::PackedVector<T, Ownership> &vec)
{ {
writer.WriteOne(vec.num_elements); writer.WriteOne(vec.num_elements);
writer.SerializeVector(vec.vec); storage::serialization::write(writer, vec.vec);
} }
template <typename EdgeDataT, storage::Ownership Ownership> template <typename EdgeDataT, storage::Ownership Ownership>
inline void read(storage::io::FileReader &reader, inline void read(storage::io::FileReader &reader,
StaticGraph<EdgeDataT, Ownership> &graph) StaticGraph<EdgeDataT, Ownership> &graph)
{ {
reader.DeserializeVector(graph.node_array); storage::serialization::read(reader, graph.node_array);
reader.DeserializeVector(graph.edge_array); storage::serialization::read(reader, graph.edge_array);
} }
template <typename EdgeDataT, storage::Ownership Ownership> template <typename EdgeDataT, storage::Ownership Ownership>
inline void write(storage::io::FileWriter &writer, inline void write(storage::io::FileWriter &writer,
const StaticGraph<EdgeDataT, Ownership> &graph) const StaticGraph<EdgeDataT, Ownership> &graph)
{ {
writer.SerializeVector(graph.node_array); storage::serialization::write(writer, graph.node_array);
writer.SerializeVector(graph.edge_array); storage::serialization::write(writer, graph.edge_array);
} }
template <typename EdgeDataT> template <typename EdgeDataT>
inline void read(storage::io::FileReader &reader, inline void read(storage::io::FileReader &reader,
DynamicGraph<EdgeDataT> &graph) DynamicGraph<EdgeDataT> &graph)
{ {
reader.DeserializeVector(graph.node_array); storage::serialization::read(reader, graph.node_array);
auto num_edges = reader.ReadElementCount64(); auto num_edges = reader.ReadElementCount64();
graph.edge_list.resize(num_edges); graph.edge_list.resize(num_edges);
for (auto index : irange<std::size_t>(0, num_edges)) for (auto index : irange<std::size_t>(0, num_edges))
@ -63,7 +65,7 @@ template <typename EdgeDataT>
inline void write(storage::io::FileWriter &writer, inline void write(storage::io::FileWriter &writer,
const DynamicGraph<EdgeDataT> &graph) const DynamicGraph<EdgeDataT> &graph)
{ {
writer.SerializeVector(graph.node_array); storage::serialization::write(writer, graph.node_array);
writer.WriteElementCount64(graph.number_of_edges); writer.WriteElementCount64(graph.number_of_edges);
for (auto index : irange<std::size_t>(0, graph.number_of_edges)) for (auto index : irange<std::size_t>(0, graph.number_of_edges))
{ {

View File

@ -4,7 +4,7 @@
#include "util/graph_traits.hpp" #include "util/graph_traits.hpp"
#include "util/integer_range.hpp" #include "util/integer_range.hpp"
#include "util/percent.hpp" #include "util/percent.hpp"
#include "util/shared_memory_vector_wrapper.hpp" #include "util/vector_view.hpp"
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
#include "storage/shared_memory_ownership.hpp" #include "storage/shared_memory_ownership.hpp"
@ -122,7 +122,7 @@ EntryT edgeToEntry(const OtherEdge &from, std::false_type)
template <typename EdgeDataT, storage::Ownership Ownership = storage::Ownership::Container> template <typename EdgeDataT, storage::Ownership Ownership = storage::Ownership::Container>
class StaticGraph class StaticGraph
{ {
template <typename T> using Vector = typename util::ShM<T, Ownership>::vector; template <typename T> using Vector = util::ViewOrVector<T, Ownership>;
public: public:
using InputEdge = static_graph_details::SortableEdgeWithData<EdgeDataT>; using InputEdge = static_graph_details::SortableEdgeWithData<EdgeDataT>;

View File

@ -9,7 +9,7 @@
#include "util/hilbert_value.hpp" #include "util/hilbert_value.hpp"
#include "util/integer_range.hpp" #include "util/integer_range.hpp"
#include "util/rectangle.hpp" #include "util/rectangle.hpp"
#include "util/shared_memory_vector_wrapper.hpp" #include "util/vector_view.hpp"
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
#include "util/web_mercator.hpp" #include "util/web_mercator.hpp"
@ -58,7 +58,7 @@ template <class EdgeDataT,
std::uint32_t LEAF_PAGE_SIZE = 4096> std::uint32_t LEAF_PAGE_SIZE = 4096>
class StaticRTree class StaticRTree
{ {
template <typename T> using Vector = typename ShM<T, Ownership>::vector; template <typename T> using Vector = ViewOrVector<T, Ownership>;
public: public:
using Rectangle = RectangleInt2D; using Rectangle = RectangleInt2D;
@ -160,7 +160,7 @@ class StaticRTree
boost::iostreams::mapped_file_source m_leaves_region; boost::iostreams::mapped_file_source m_leaves_region;
// read-only view of leaves // read-only view of leaves
typename ShM<const LeafNode, storage::Ownership::View>::vector m_leaves; util::vector_view<const LeafNode> m_leaves;
public: public:
StaticRTree(const StaticRTree &) = delete; StaticRTree(const StaticRTree &) = delete;

View File

@ -1,5 +1,5 @@
#ifndef SHARED_MEMORY_VECTOR_WRAPPER_HPP #ifndef UTIL_VECTOR_VIEW_HPP
#define SHARED_MEMORY_VECTOR_WRAPPER_HPP #define UTIL_VECTOR_VIEW_HPP
#include "util/exception.hpp" #include "util/exception.hpp"
#include "util/log.hpp" #include "util/log.hpp"
@ -25,11 +25,13 @@ namespace util
{ {
template <typename DataT> template <typename DataT>
class ShMemIterator class VectorViewIterator : public boost::iterator_facade<VectorViewIterator<DataT>,
: public boost::iterator_facade<ShMemIterator<DataT>, DataT, boost::random_access_traversal_tag> DataT,
boost::random_access_traversal_tag>
{ {
typedef boost::iterator_facade<ShMemIterator<DataT>, DataT, boost::random_access_traversal_tag> typedef boost::
base_t; iterator_facade<VectorViewIterator<DataT>, DataT, boost::random_access_traversal_tag>
base_t;
public: public:
typedef typename base_t::value_type value_type; typedef typename base_t::value_type value_type;
@ -37,16 +39,16 @@ class ShMemIterator
typedef typename base_t::reference reference; typedef typename base_t::reference reference;
typedef std::random_access_iterator_tag iterator_category; typedef std::random_access_iterator_tag iterator_category;
explicit ShMemIterator() : m_value(nullptr) {} explicit VectorViewIterator() : m_value(nullptr) {}
explicit ShMemIterator(DataT *x) : m_value(x) {} explicit VectorViewIterator(DataT *x) : m_value(x) {}
private: private:
void increment() { ++m_value; } void increment() { ++m_value; }
void decrement() { --m_value; } void decrement() { --m_value; }
void advance(difference_type offset) { m_value += offset; } void advance(difference_type offset) { m_value += offset; }
bool equal(const ShMemIterator &other) const { return m_value == other.m_value; } bool equal(const VectorViewIterator &other) const { return m_value == other.m_value; }
reference dereference() const { return *m_value; } reference dereference() const { return *m_value; }
difference_type distance_to(const ShMemIterator &other) const difference_type distance_to(const VectorViewIterator &other) const
{ {
return other.m_value - m_value; return other.m_value - m_value;
} }
@ -63,8 +65,8 @@ template <typename DataT> class vector_view
public: public:
using value_type = DataT; using value_type = DataT;
using iterator = ShMemIterator<DataT>; using iterator = VectorViewIterator<DataT>;
using const_iterator = ShMemIterator<const DataT>; using const_iterator = VectorViewIterator<const DataT>;
using reverse_iterator = boost::reverse_iterator<iterator>; using reverse_iterator = boost::reverse_iterator<iterator>;
vector_view() : m_ptr(nullptr), m_size(0) {} vector_view() : m_ptr(nullptr), m_size(0) {}
@ -88,7 +90,9 @@ template <typename DataT> class vector_view
{ {
if (m_size != size) if (m_size != size)
{ {
throw util::exception("Invalid resize " + std::to_string(size) + " on immutable vector view of size " + std::to_string(m_size) + "."); throw util::exception("Invalid resize " + std::to_string(size) +
" on immutable vector view of size " + std::to_string(m_size) +
".");
} }
} }
@ -189,12 +193,10 @@ template <typename DataT> void swap(vector_view<DataT> &lhs, vector_view<DataT>
std::swap(lhs.m_size, rhs.m_size); std::swap(lhs.m_size, rhs.m_size);
} }
template <typename DataT, storage::Ownership Ownership> struct ShM template <typename DataT, storage::Ownership Ownership>
{ using ViewOrVector = typename std::conditional<Ownership == storage::Ownership::View,
using vector = typename std::conditional<Ownership == storage::Ownership::View, vector_view<DataT>,
vector_view<DataT>, std::vector<DataT>>::type;
std::vector<DataT>>::type;
};
} }
} }

View File

@ -1,6 +1,6 @@
#include "contractor/contractor.hpp" #include "contractor/contractor.hpp"
#include "contractor/files.hpp"
#include "contractor/crc32_processor.hpp" #include "contractor/crc32_processor.hpp"
#include "contractor/files.hpp"
#include "contractor/graph_contractor.hpp" #include "contractor/graph_contractor.hpp"
#include "contractor/graph_contractor_adaptors.hpp" #include "contractor/graph_contractor_adaptors.hpp"
@ -47,9 +47,9 @@ int Contractor::Run()
util::Log() << "Reading node weights."; util::Log() << "Reading node weights.";
std::vector<EdgeWeight> node_weights; std::vector<EdgeWeight> node_weights;
{ {
storage::io::FileReader node_file(config.node_file_path, storage::io::FileReader reader(config.node_file_path,
storage::io::FileReader::VerifyFingerprint); storage::io::FileReader::VerifyFingerprint);
node_file.DeserializeVector(node_weights); storage::serialization::read(reader, node_weights);
} }
util::Log() << "Done reading node weights."; util::Log() << "Done reading node weights.";
@ -115,10 +115,10 @@ void Contractor::WriteNodeLevels(std::vector<float> &&in_node_levels) const
{ {
std::vector<float> node_levels(std::move(in_node_levels)); std::vector<float> node_levels(std::move(in_node_levels));
storage::io::FileWriter node_level_file(config.level_output_path, storage::io::FileWriter writer(config.level_output_path,
storage::io::FileWriter::HasNoFingerprint); storage::io::FileWriter::HasNoFingerprint);
node_level_file.SerializeVector(node_levels); storage::serialization::write(writer, node_levels);
} }
void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const
@ -138,9 +138,8 @@ void Contractor::WriteCoreNodeMarker(std::vector<bool> &&in_is_core_node) const
core_marker_output_file.WriteFrom(unpacked_bool_flags.data(), count); core_marker_output_file.WriteFrom(unpacked_bool_flags.data(), count);
} }
void void Contractor::WriteContractedGraph(unsigned max_node_id,
Contractor::WriteContractedGraph(unsigned max_node_id, util::DeallocatingVector<QueryEdge> contracted_edge_list)
util::DeallocatingVector<QueryEdge> contracted_edge_list)
{ {
// Sorting contracted edges in a way that the static query graph can read some in in-place. // Sorting contracted edges in a way that the static query graph can read some in in-place.
tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end()); tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end());
@ -150,7 +149,7 @@ Contractor::WriteContractedGraph(unsigned max_node_id,
RangebasedCRC32 crc32_calculator; RangebasedCRC32 crc32_calculator;
const unsigned checksum = crc32_calculator(contracted_edge_list); const unsigned checksum = crc32_calculator(contracted_edge_list);
QueryGraph query_graph {max_node_id+1, contracted_edge_list}; QueryGraph query_graph{max_node_id + 1, contracted_edge_list};
files::writeGraph(config.graph_output_path, checksum, query_graph); files::writeGraph(config.graph_output_path, checksum, query_graph);
} }

View File

@ -48,8 +48,7 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory(
std::vector<std::uint32_t> &turn_lane_offsets, std::vector<std::uint32_t> &turn_lane_offsets,
std::vector<guidance::TurnLaneType::Mask> &turn_lane_masks, std::vector<guidance::TurnLaneType::Mask> &turn_lane_masks,
guidance::LaneDescriptionMap &lane_description_map) guidance::LaneDescriptionMap &lane_description_map)
: m_max_edge_id(0), m_coordinates(coordinates), : m_max_edge_id(0), m_coordinates(coordinates), m_osm_node_ids(osm_node_ids),
m_osm_node_ids(osm_node_ids),
m_node_based_graph(std::move(node_based_graph)), m_node_based_graph(std::move(node_based_graph)),
m_restriction_map(std::move(restriction_map)), m_barrier_nodes(barrier_nodes), m_restriction_map(std::move(restriction_map)), m_barrier_nodes(barrier_nodes),
m_traffic_lights(traffic_lights), m_compressed_edge_container(compressed_edge_container), m_traffic_lights(traffic_lights), m_compressed_edge_container(compressed_edge_container),
@ -540,16 +539,14 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
const bool isTrivial = m_compressed_edge_container.IsTrivial(incoming_edge); const bool isTrivial = m_compressed_edge_container.IsTrivial(incoming_edge);
const auto &from_node = const auto &from_node =
isTrivial isTrivial ? m_osm_node_ids[node_along_road_entering]
? m_osm_node_ids[node_along_road_entering] : m_osm_node_ids[m_compressed_edge_container.GetLastEdgeSourceID(
: m_osm_node_ids[m_compressed_edge_container.GetLastEdgeSourceID( incoming_edge)];
incoming_edge)];
const auto &via_node = const auto &via_node =
m_osm_node_ids[m_compressed_edge_container.GetLastEdgeTargetID( m_osm_node_ids[m_compressed_edge_container.GetLastEdgeTargetID(
incoming_edge)]; incoming_edge)];
const auto &to_node = const auto &to_node =
m_osm_node_ids[m_compressed_edge_container.GetFirstEdgeTargetID( m_osm_node_ids[m_compressed_edge_container.GetFirstEdgeTargetID(turn.eid)];
turn.eid)];
lookup::TurnIndexBlock turn_index_block = {from_node, via_node, to_node}; lookup::TurnIndexBlock turn_index_block = {from_node, via_node, to_node};
@ -562,32 +559,32 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges(
// write weight penalties per turn // write weight penalties per turn
BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size()); BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size());
{ {
storage::io::FileWriter turn_weight_penalties_file( storage::io::FileWriter writer(turn_weight_penalties_filename,
turn_weight_penalties_filename, storage::io::FileWriter::HasNoFingerprint); storage::io::FileWriter::HasNoFingerprint);
turn_weight_penalties_file.SerializeVector(turn_weight_penalties); storage::serialization::write(writer, turn_weight_penalties);
} }
{ {
storage::io::FileWriter turn_duration_penalties_file( storage::io::FileWriter writer(turn_duration_penalties_filename,
turn_duration_penalties_filename, storage::io::FileWriter::HasNoFingerprint); storage::io::FileWriter::HasNoFingerprint);
turn_duration_penalties_file.SerializeVector(turn_duration_penalties); storage::serialization::write(writer, turn_duration_penalties);
} }
util::Log() << "Created " << entry_class_hash.size() << " entry classes and " util::Log() << "Created " << entry_class_hash.size() << " entry classes and "
<< bearing_class_hash.size() << " Bearing Classes"; << bearing_class_hash.size() << " Bearing Classes";
util::Log() << "Writing Turn Lane Data to File..."; util::Log() << "Writing Turn Lane Data to File...";
{
storage::io::FileWriter writer(turn_lane_data_filename,
storage::io::FileWriter::HasNoFingerprint);
storage::io::FileWriter turn_lane_data_file(turn_lane_data_filename, std::vector<util::guidance::LaneTupleIdPair> lane_data(lane_data_map.size());
storage::io::FileWriter::HasNoFingerprint); // extract lane data sorted by ID
for (auto itr : lane_data_map)
std::vector<util::guidance::LaneTupleIdPair> lane_data(lane_data_map.size()); lane_data[itr.second] = itr.first;
// extract lane data sorted by ID
for (auto itr : lane_data_map)
lane_data[itr.second] = itr.first;
turn_lane_data_file.SerializeVector(lane_data);
storage::serialization::write(writer, lane_data);
}
util::Log() << "done."; util::Log() << "done.";
files::writeTurnData(turn_data_filename, turn_data_container); files::writeTurnData(turn_data_filename, turn_data_container);

View File

@ -281,11 +281,11 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
util::Log() << "Saving edge-based node weights to file."; util::Log() << "Saving edge-based node weights to file.";
TIMER_START(timer_write_node_weights); TIMER_START(timer_write_node_weights);
{
storage::io::FileWriter edge_file(config.edge_based_node_weights_output_path, storage::io::FileWriter writer(config.edge_based_node_weights_output_path,
storage::io::FileWriter::GenerateFingerprint); storage::io::FileWriter::GenerateFingerprint);
edge_file.SerializeVector(edge_based_node_weights); storage::serialization::write(writer, edge_based_node_weights);
}
TIMER_STOP(timer_write_node_weights); TIMER_STOP(timer_write_node_weights);
util::Log() << "Done writing. (" << TIMER_SEC(timer_write_node_weights) << ")"; util::Log() << "Done writing. (" << TIMER_SEC(timer_write_node_weights) << ")";
@ -299,7 +299,8 @@ int Extractor::run(ScriptingEnvironment &scripting_environment)
TIMER_STOP(rtree); TIMER_STOP(rtree);
util::Log() << "Writing node map ..."; util::Log() << "Writing node map ...";
files::writeNodes<storage::Ownership::Container>(config.node_output_path, coordinates, osm_node_ids); files::writeNodes<storage::Ownership::Container>(
config.node_output_path, coordinates, osm_node_ids);
WriteEdgeBasedGraph(config.edge_graph_output_path, max_edge_id, edge_based_edge_list); WriteEdgeBasedGraph(config.edge_graph_output_path, max_edge_id, edge_based_edge_list);
@ -509,9 +510,8 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
}; };
compressed_node_based_graph_writing = std::async(std::launch::async, [&] { compressed_node_based_graph_writing = std::async(std::launch::async, [&] {
WriteCompressedNodeBasedGraph(config.compressed_node_based_graph_output_path, WriteCompressedNodeBasedGraph(
*node_based_graph, config.compressed_node_based_graph_output_path, *node_based_graph, coordinates);
coordinates);
}); });
WriteTurnLaneData(config.turn_lane_descriptions_file_name); WriteTurnLaneData(config.turn_lane_descriptions_file_name);
@ -612,11 +612,11 @@ void Extractor::WriteIntersectionClassificationData(
const std::vector<util::guidance::BearingClass> &bearing_classes, const std::vector<util::guidance::BearingClass> &bearing_classes,
const std::vector<util::guidance::EntryClass> &entry_classes) const const std::vector<util::guidance::EntryClass> &entry_classes) const
{ {
storage::io::FileWriter file(output_file_name, storage::io::FileWriter::GenerateFingerprint); storage::io::FileWriter writer(output_file_name, storage::io::FileWriter::GenerateFingerprint);
util::Log() << "Writing Intersection Classification Data"; util::Log() << "Writing Intersection Classification Data";
TIMER_START(write_edges); TIMER_START(write_edges);
file.SerializeVector(node_based_intersection_classes); storage::serialization::write(writer, node_based_intersection_classes);
// create range table for vectors: // create range table for vectors:
std::vector<unsigned> bearing_counts; std::vector<unsigned> bearing_counts;
@ -630,17 +630,17 @@ void Extractor::WriteIntersectionClassificationData(
} }
util::RangeTable<> bearing_class_range_table(bearing_counts); util::RangeTable<> bearing_class_range_table(bearing_counts);
bearing_class_range_table.Write(file); bearing_class_range_table.Write(writer);
file.WriteOne(total_bearings); writer.WriteOne(total_bearings);
for (const auto &bearing_class : bearing_classes) for (const auto &bearing_class : bearing_classes)
{ {
const auto &bearings = bearing_class.getAvailableBearings(); const auto &bearings = bearing_class.getAvailableBearings();
file.WriteFrom(bearings.data(), bearings.size()); writer.WriteFrom(bearings.data(), bearings.size());
} }
file.SerializeVector(entry_classes); storage::serialization::write(writer, entry_classes);
TIMER_STOP(write_edges); TIMER_STOP(write_edges);
util::Log() << "ok, after " << TIMER_SEC(write_edges) << "s for " util::Log() << "ok, after " << TIMER_SEC(write_edges) << "s for "
@ -659,9 +659,9 @@ void Extractor::WriteTurnLaneData(const std::string &turn_lane_file) const
util::Log() << "Writing turn lane masks..."; util::Log() << "Writing turn lane masks...";
TIMER_START(turn_lane_timer); TIMER_START(turn_lane_timer);
storage::io::FileWriter file(turn_lane_file, storage::io::FileWriter::HasNoFingerprint); storage::io::FileWriter writer(turn_lane_file, storage::io::FileWriter::HasNoFingerprint);
file.SerializeVector(turn_lane_offsets); storage::serialization::write(writer, turn_lane_offsets);
file.SerializeVector(turn_lane_masks); storage::serialization::write(writer, turn_lane_masks);
TIMER_STOP(turn_lane_timer); TIMER_STOP(turn_lane_timer);
util::Log() << "done (" << TIMER_SEC(turn_lane_timer) << ")"; util::Log() << "done (" << TIMER_SEC(turn_lane_timer) << ")";

View File

@ -34,10 +34,10 @@
#include "util/log.hpp" #include "util/log.hpp"
#include "util/packed_vector.hpp" #include "util/packed_vector.hpp"
#include "util/range_table.hpp" #include "util/range_table.hpp"
#include "util/shared_memory_vector_wrapper.hpp"
#include "util/static_graph.hpp" #include "util/static_graph.hpp"
#include "util/static_rtree.hpp" #include "util/static_rtree.hpp"
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
#include "util/vector_view.hpp"
#ifdef __linux__ #ifdef __linux__
#include <sys/mman.h> #include <sys/mman.h>
@ -63,7 +63,7 @@ namespace storage
{ {
using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf; using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf;
using RTreeNode = util:: StaticRTree<RTreeLeaf, storage::Ownership::View>::TreeNode; using RTreeNode = util::StaticRTree<RTreeLeaf, storage::Ownership::View>::TreeNode;
using QueryGraph = util::StaticGraph<contractor::QueryEdge::EdgeData>; using QueryGraph = util::StaticGraph<contractor::QueryEdge::EdgeData>;
using EdgeBasedGraph = util::StaticGraph<extractor::EdgeBasedEdge::EdgeData>; using EdgeBasedGraph = util::StaticGraph<extractor::EdgeBasedEdge::EdgeData>;
@ -361,7 +361,7 @@ void Storage::PopulateLayout(DataLayout &layout)
io::FileReader::VerifyFingerprint); io::FileReader::VerifyFingerprint);
std::vector<BearingClassID> bearing_class_id_table; std::vector<BearingClassID> bearing_class_id_table;
intersection_file.DeserializeVector(bearing_class_id_table); serialization::read(intersection_file, bearing_class_id_table);
layout.SetBlockSize<BearingClassID>(DataLayout::BEARING_CLASSID, layout.SetBlockSize<BearingClassID>(DataLayout::BEARING_CLASSID,
bearing_class_id_table.size()); bearing_class_id_table.size());
@ -386,7 +386,7 @@ void Storage::PopulateLayout(DataLayout &layout)
layout.SetBlockSize<DiscreteBearing>(DataLayout::BEARING_VALUES, num_bearings); layout.SetBlockSize<DiscreteBearing>(DataLayout::BEARING_VALUES, num_bearings);
std::vector<util::guidance::EntryClass> entry_class_table; std::vector<util::guidance::EntryClass> entry_class_table;
intersection_file.DeserializeVector(entry_class_table); serialization::read(intersection_file, entry_class_table);
layout.SetBlockSize<util::guidance::EntryClass>(DataLayout::ENTRY_CLASS, layout.SetBlockSize<util::guidance::EntryClass>(DataLayout::ENTRY_CLASS,
entry_class_table.size()); entry_class_table.size());
@ -690,11 +690,13 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
layout.GetBlockPtr<util::Coordinate, true>(memory_ptr, DataLayout::COORDINATE_LIST); layout.GetBlockPtr<util::Coordinate, true>(memory_ptr, DataLayout::COORDINATE_LIST);
const auto osmnodeid_ptr = const auto osmnodeid_ptr =
layout.GetBlockPtr<std::uint64_t, true>(memory_ptr, DataLayout::OSM_NODE_ID_LIST); layout.GetBlockPtr<std::uint64_t, true>(memory_ptr, DataLayout::OSM_NODE_ID_LIST);
util::vector_view<util::Coordinate> coordinates(coordinates_ptr, layout.num_entries[DataLayout::COORDINATE_LIST]); util::vector_view<util::Coordinate> coordinates(
coordinates_ptr, layout.num_entries[DataLayout::COORDINATE_LIST]);
util::PackedVectorView<OSMNodeID> osm_node_ids; util::PackedVectorView<OSMNodeID> osm_node_ids;
osm_node_ids.reset(osmnodeid_ptr, layout.num_entries[DataLayout::OSM_NODE_ID_LIST]); osm_node_ids.reset(osmnodeid_ptr, layout.num_entries[DataLayout::OSM_NODE_ID_LIST]);
extractor::files::readNodes<storage::Ownership::View>(config.nodes_data_path, coordinates, osm_node_ids); extractor::files::readNodes<storage::Ownership::View>(
config.nodes_data_path, coordinates, osm_node_ids);
} }
// load turn weight penalties // load turn weight penalties
@ -790,7 +792,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
io::FileReader::VerifyFingerprint); io::FileReader::VerifyFingerprint);
std::vector<BearingClassID> bearing_class_id_table; std::vector<BearingClassID> bearing_class_id_table;
intersection_file.DeserializeVector(bearing_class_id_table); serialization::read(intersection_file, bearing_class_id_table);
const auto bearing_blocks = intersection_file.ReadElementCount32(); const auto bearing_blocks = intersection_file.ReadElementCount32();
intersection_file.Skip<std::uint32_t>(1); // sum_lengths intersection_file.Skip<std::uint32_t>(1); // sum_lengths
@ -808,7 +810,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
intersection_file.ReadInto(bearing_class_table.data(), num_bearings); intersection_file.ReadInto(bearing_class_table.data(), num_bearings);
std::vector<util::guidance::EntryClass> entry_class_table; std::vector<util::guidance::EntryClass> entry_class_table;
intersection_file.DeserializeVector(entry_class_table); serialization::read(intersection_file, entry_class_table);
// load intersection classes // load intersection classes
if (!bearing_class_id_table.empty()) if (!bearing_class_id_table.empty())

View File

@ -6,7 +6,6 @@
#include "extractor/edge_based_graph_factory.hpp" #include "extractor/edge_based_graph_factory.hpp"
#include "extractor/files.hpp" #include "extractor/files.hpp"
#include "extractor/node_based_edge.hpp" #include "extractor/node_based_edge.hpp"
#include "extractor/seralization.hpp"
#include "storage/io.hpp" #include "storage/io.hpp"
@ -460,14 +459,14 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
const auto load_turn_weight_penalties = [&] { const auto load_turn_weight_penalties = [&] {
using storage::io::FileReader; using storage::io::FileReader;
FileReader file(config.turn_weight_penalties_path, FileReader::HasNoFingerprint); FileReader reader(config.turn_weight_penalties_path, FileReader::HasNoFingerprint);
file.DeserializeVector(turn_weight_penalties); storage::serialization::read(reader, turn_weight_penalties);
}; };
const auto load_turn_duration_penalties = [&] { const auto load_turn_duration_penalties = [&] {
using storage::io::FileReader; using storage::io::FileReader;
FileReader file(config.turn_duration_penalties_path, FileReader::HasNoFingerprint); FileReader reader(config.turn_duration_penalties_path, FileReader::HasNoFingerprint);
file.DeserializeVector(turn_duration_penalties); storage::serialization::read(reader, turn_duration_penalties);
}; };
const auto load_profile_properties = [&] { const auto load_profile_properties = [&] {
@ -648,8 +647,8 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
if (update_turn_penalties) if (update_turn_penalties)
{ {
const auto save_penalties = [](const auto &filename, const auto &data) -> void { const auto save_penalties = [](const auto &filename, const auto &data) -> void {
storage::io::FileWriter file(filename, storage::io::FileWriter::HasNoFingerprint); storage::io::FileWriter writer(filename, storage::io::FileWriter::HasNoFingerprint);
file.SerializeVector(data); storage::serialization::write(writer, data);
}; };
tbb::parallel_invoke( tbb::parallel_invoke(

View File

@ -1,4 +1,5 @@
#include "storage/io.hpp" #include "storage/io.hpp"
#include "storage/serialization.hpp"
#include "util/exception.hpp" #include "util/exception.hpp"
#include "util/fingerprint.hpp" #include "util/fingerprint.hpp"
#include "util/typedefs.hpp" #include "util/typedefs.hpp"
@ -29,12 +30,12 @@ BOOST_AUTO_TEST_CASE(io_data)
{ {
osrm::storage::io::FileWriter outfile(IO_TMP_FILE, osrm::storage::io::FileWriter outfile(IO_TMP_FILE,
osrm::storage::io::FileWriter::GenerateFingerprint); osrm::storage::io::FileWriter::GenerateFingerprint);
outfile.SerializeVector(data_in); osrm::storage::serialization::write(outfile, data_in);
} }
osrm::storage::io::FileReader infile(IO_TMP_FILE, osrm::storage::io::FileReader infile(IO_TMP_FILE,
osrm::storage::io::FileReader::VerifyFingerprint); osrm::storage::io::FileReader::VerifyFingerprint);
infile.DeserializeVector(data_out); osrm::storage::serialization::read(infile, data_out);
BOOST_REQUIRE_EQUAL(data_in.size(), data_out.size()); BOOST_REQUIRE_EQUAL(data_in.size(), data_out.size());
BOOST_CHECK_EQUAL_COLLECTIONS(data_out.begin(), data_out.end(), data_in.begin(), data_in.end()); BOOST_CHECK_EQUAL_COLLECTIONS(data_out.begin(), data_out.end(), data_in.begin(), data_in.end());
@ -65,7 +66,7 @@ BOOST_AUTO_TEST_CASE(file_too_small)
{ {
osrm::storage::io::FileWriter outfile( osrm::storage::io::FileWriter outfile(
IO_TOO_SMALL_FILE, osrm::storage::io::FileWriter::GenerateFingerprint); IO_TOO_SMALL_FILE, osrm::storage::io::FileWriter::GenerateFingerprint);
outfile.SerializeVector(v); osrm::storage::serialization::write(outfile, v);
} }
std::fstream f(IO_TOO_SMALL_FILE); std::fstream f(IO_TOO_SMALL_FILE);
@ -79,7 +80,7 @@ BOOST_AUTO_TEST_CASE(file_too_small)
osrm::storage::io::FileReader infile(IO_TOO_SMALL_FILE, osrm::storage::io::FileReader infile(IO_TOO_SMALL_FILE,
osrm::storage::io::FileReader::VerifyFingerprint); osrm::storage::io::FileReader::VerifyFingerprint);
std::vector<int> buffer; std::vector<int> buffer;
infile.DeserializeVector(buffer); osrm::storage::serialization::read(infile, buffer);
BOOST_REQUIRE_MESSAGE(false, "Should not get here"); BOOST_REQUIRE_MESSAGE(false, "Should not get here");
} }
catch (const osrm::util::exception &e) catch (const osrm::util::exception &e)
@ -101,7 +102,7 @@ BOOST_AUTO_TEST_CASE(io_corrupt_fingerprint)
osrm::storage::io::FileWriter::HasNoFingerprint); osrm::storage::io::FileWriter::HasNoFingerprint);
outfile.WriteOne(0xDEADBEEFCAFEFACE); outfile.WriteOne(0xDEADBEEFCAFEFACE);
outfile.SerializeVector(v); osrm::storage::serialization::write(outfile, v);
} }
try try
@ -130,7 +131,7 @@ BOOST_AUTO_TEST_CASE(io_incompatible_fingerprint)
const auto fingerprint = osrm::util::FingerPrint::GetValid(); const auto fingerprint = osrm::util::FingerPrint::GetValid();
outfile.WriteOne(fingerprint); outfile.WriteOne(fingerprint);
outfile.SerializeVector(v); osrm::storage::serialization::write(outfile, v);
} }
std::fstream f(IO_INCOMPATIBLE_FINGERPRINT_FILE); std::fstream f(IO_INCOMPATIBLE_FINGERPRINT_FILE);