From d94017dfae993172868a942703457ead1ca28ec8 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 4 Apr 2017 22:00:17 +0000 Subject: [PATCH] No shared_memory_wrapper, (De)SerializeVector --- include/extractor/files.hpp | 30 +++--- include/extractor/segment_data_container.hpp | 4 +- include/extractor/seralization.hpp | 97 ------------------- include/extractor/serialization.hpp | 88 +++++++++++++++++ include/extractor/turn_data_container.hpp | 4 +- include/partition/cell_storage.hpp | 4 +- include/partition/multi_level_graph.hpp | 3 +- include/partition/multi_level_partition.hpp | 4 +- include/partition/serialization.hpp | 41 ++++---- include/storage/io.hpp | 70 ------------- include/storage/serialization.hpp | 47 +++++++++ include/util/name_table.hpp | 1 - include/util/packed_vector.hpp | 4 +- include/util/range_table.hpp | 6 +- include/util/serialization.hpp | 18 ++-- include/util/static_graph.hpp | 4 +- include/util/static_rtree.hpp | 6 +- ...ory_vector_wrapper.hpp => vector_view.hpp} | 40 ++++---- src/contractor/contractor.cpp | 19 ++-- src/extractor/edge_based_graph_factory.cpp | 43 ++++---- src/extractor/extractor.cpp | 36 +++---- src/storage/storage.cpp | 18 ++-- src/updater/updater.cpp | 13 ++- unit_tests/util/io.cpp | 13 +-- 24 files changed, 292 insertions(+), 321 deletions(-) delete mode 100644 include/extractor/seralization.hpp create mode 100644 include/extractor/serialization.hpp create mode 100644 include/storage/serialization.hpp rename include/util/{shared_memory_vector_wrapper.hpp => vector_view.hpp} (78%) diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index a15f43cce..a34899677 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -2,7 +2,7 @@ #define OSRM_EXTRACTOR_FILES_HPP #include "extractor/guidance/turn_lane_types.hpp" -#include "extractor/seralization.hpp" +#include "extractor/serialization.hpp" #include "util/coordinate.hpp" #include "util/packed_vector.hpp" @@ -20,26 +20,26 @@ namespace files // reads .osrm.nodes template inline void readNodes(const boost::filesystem::path &path, - typename util::ShM::vector &coordinates, + util::ViewOrVector &coordinates, util::detail::PackedVector &osm_node_ids) { const auto fingerprint = storage::io::FileReader::VerifyFingerprint; storage::io::FileReader reader{path, fingerprint}; - reader.DeserializeVector(coordinates); + storage::serialization::read(reader, coordinates); util::serialization::read(reader, osm_node_ids); } // writes .osrm.nodes template inline void writeNodes(const boost::filesystem::path &path, - const typename util::ShM::vector &coordinates, + const util::ViewOrVector &coordinates, const util::detail::PackedVector &osm_node_ids) { const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; storage::io::FileWriter writer{path, fingerprint}; - writer.SerializeVector(coordinates); + storage::serialization::write(writer, coordinates); util::serialization::write(writer, osm_node_ids); } @@ -49,7 +49,7 @@ inline void readNBGMapping(const boost::filesystem::path &path, std::vector inline void readTurnLaneDescriptions(const boost::filesystem::path &path, - typename util::ShM::vector &turn_offsets, - typename util::ShM::vector &turn_masks) + util::ViewOrVector &turn_offsets, + util::ViewOrVector &turn_masks) { const auto fingerprint = storage::io::FileReader::HasNoFingerprint; storage::io::FileReader reader{path, fingerprint}; - reader.DeserializeVector(turn_offsets); - reader.DeserializeVector(turn_masks); + storage::serialization::read(reader, turn_offsets); + storage::serialization::read(reader, turn_masks); } // writes .osrm.tls template inline void writeTurnLaneDescriptions(const boost::filesystem::path &path, - const typename util::ShM::vector &turn_offsets, - const typename util::ShM::vector &turn_masks) + const util::ViewOrVector &turn_offsets, + const util::ViewOrVector &turn_masks) { const auto fingerprint = storage::io::FileWriter::HasNoFingerprint; storage::io::FileWriter writer{path, fingerprint}; - writer.SerializeVector(turn_offsets); - writer.SerializeVector(turn_masks); + storage::serialization::write(writer, turn_offsets); + storage::serialization::write(writer, turn_masks); } } diff --git a/include/extractor/segment_data_container.hpp b/include/extractor/segment_data_container.hpp index 8e677855a..c6633a3a2 100644 --- a/include/extractor/segment_data_container.hpp +++ b/include/extractor/segment_data_container.hpp @@ -1,7 +1,7 @@ #ifndef OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_ #define OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_ -#include "util/shared_memory_vector_wrapper.hpp" +#include "util/vector_view.hpp" #include "util/typedefs.hpp" #include "storage/shared_memory_ownership.hpp" @@ -50,7 +50,7 @@ namespace detail { template class SegmentDataContainerImpl { - template using Vector = typename util::ShM::vector; + template using Vector = util::ViewOrVector; friend CompressedEdgeContainer; diff --git a/include/extractor/seralization.hpp b/include/extractor/seralization.hpp deleted file mode 100644 index 0a63fe63d..000000000 --- a/include/extractor/seralization.hpp +++ /dev/null @@ -1,97 +0,0 @@ -#ifndef OSRM_EXTRACTOR_IO_HPP -#define OSRM_EXTRACTOR_IO_HPP - -#include "extractor/datasources.hpp" -#include "extractor/nbg_to_ebg.hpp" -#include "extractor/segment_data_container.hpp" -#include "extractor/turn_data_container.hpp" - -#include "storage/io.hpp" - -#include - -namespace osrm -{ -namespace extractor -{ -namespace serialization -{ - -inline void read(storage::io::FileReader &reader, std::vector &mapping) -{ - reader.DeserializeVector(mapping); -} - -inline void write(storage::io::FileWriter &writer, const std::vector &mapping) -{ - writer.SerializeVector(mapping); -} - -inline void read(storage::io::FileReader &reader, Datasources &sources) -{ - reader.ReadInto(sources); -} - -inline void write(storage::io::FileWriter &writer, Datasources &sources) -{ - writer.WriteFrom(sources); -} - -template -inline void read(storage::io::FileReader &reader, - detail::SegmentDataContainerImpl &segment_data) -{ - reader.DeserializeVector(segment_data.index); - reader.DeserializeVector(segment_data.nodes); - reader.DeserializeVector(segment_data.fwd_weights); - reader.DeserializeVector(segment_data.rev_weights); - reader.DeserializeVector(segment_data.fwd_durations); - reader.DeserializeVector(segment_data.rev_durations); - reader.DeserializeVector(segment_data.datasources); -} - -template -inline void write(storage::io::FileWriter &writer, - const detail::SegmentDataContainerImpl &segment_data) -{ - writer.SerializeVector(segment_data.index); - writer.SerializeVector(segment_data.nodes); - writer.SerializeVector(segment_data.fwd_weights); - writer.SerializeVector(segment_data.rev_weights); - writer.SerializeVector(segment_data.fwd_durations); - writer.SerializeVector(segment_data.rev_durations); - writer.SerializeVector(segment_data.datasources); -} - -template -inline void read(storage::io::FileReader &reader, - detail::TurnDataContainerImpl &turn_data_container) -{ - reader.DeserializeVector(turn_data_container.geometry_ids); - reader.DeserializeVector(turn_data_container.name_ids); - reader.DeserializeVector(turn_data_container.turn_instructions); - reader.DeserializeVector(turn_data_container.lane_data_ids); - reader.DeserializeVector(turn_data_container.travel_modes); - reader.DeserializeVector(turn_data_container.entry_class_ids); - reader.DeserializeVector(turn_data_container.pre_turn_bearings); - reader.DeserializeVector(turn_data_container.post_turn_bearings); -} - -template -inline void write(storage::io::FileWriter &writer, - const detail::TurnDataContainerImpl &turn_data_container) -{ - writer.SerializeVector(turn_data_container.geometry_ids); - writer.SerializeVector(turn_data_container.name_ids); - writer.SerializeVector(turn_data_container.turn_instructions); - writer.SerializeVector(turn_data_container.lane_data_ids); - writer.SerializeVector(turn_data_container.travel_modes); - writer.SerializeVector(turn_data_container.entry_class_ids); - writer.SerializeVector(turn_data_container.pre_turn_bearings); - writer.SerializeVector(turn_data_container.post_turn_bearings); -} -} -} -} - -#endif diff --git a/include/extractor/serialization.hpp b/include/extractor/serialization.hpp new file mode 100644 index 000000000..2f297ccdd --- /dev/null +++ b/include/extractor/serialization.hpp @@ -0,0 +1,88 @@ +#ifndef OSRM_EXTRACTOR_IO_HPP +#define OSRM_EXTRACTOR_IO_HPP + +#include "extractor/datasources.hpp" +#include "extractor/nbg_to_ebg.hpp" +#include "extractor/segment_data_container.hpp" +#include "extractor/turn_data_container.hpp" + +#include "storage/io.hpp" +#include "storage/serialization.hpp" + +#include + +namespace osrm +{ +namespace extractor +{ +namespace serialization +{ + +inline void read(storage::io::FileReader &reader, Datasources &sources) +{ + reader.ReadInto(sources); +} + +inline void write(storage::io::FileWriter &writer, Datasources &sources) +{ + writer.WriteFrom(sources); +} + +template +inline void read(storage::io::FileReader &reader, + detail::SegmentDataContainerImpl &segment_data) +{ + storage::serialization::read(reader, segment_data.index); + storage::serialization::read(reader, segment_data.nodes); + storage::serialization::read(reader, segment_data.fwd_weights); + storage::serialization::read(reader, segment_data.rev_weights); + storage::serialization::read(reader, segment_data.fwd_durations); + storage::serialization::read(reader, segment_data.rev_durations); + storage::serialization::read(reader, segment_data.datasources); +} + +template +inline void write(storage::io::FileWriter &writer, + const detail::SegmentDataContainerImpl &segment_data) +{ + storage::serialization::write(writer, segment_data.index); + storage::serialization::write(writer, segment_data.nodes); + storage::serialization::write(writer, segment_data.fwd_weights); + storage::serialization::write(writer, segment_data.rev_weights); + storage::serialization::write(writer, segment_data.fwd_durations); + storage::serialization::write(writer, segment_data.rev_durations); + storage::serialization::write(writer, segment_data.datasources); +} + +template +inline void read(storage::io::FileReader &reader, + detail::TurnDataContainerImpl &turn_data_container) +{ + storage::serialization::read(reader, turn_data_container.geometry_ids); + storage::serialization::read(reader, turn_data_container.name_ids); + storage::serialization::read(reader, turn_data_container.turn_instructions); + storage::serialization::read(reader, turn_data_container.lane_data_ids); + storage::serialization::read(reader, turn_data_container.travel_modes); + storage::serialization::read(reader, turn_data_container.entry_class_ids); + storage::serialization::read(reader, turn_data_container.pre_turn_bearings); + storage::serialization::read(reader, turn_data_container.post_turn_bearings); +} + +template +inline void write(storage::io::FileWriter &writer, + const detail::TurnDataContainerImpl &turn_data_container) +{ + storage::serialization::write(writer, turn_data_container.geometry_ids); + storage::serialization::write(writer, turn_data_container.name_ids); + storage::serialization::write(writer, turn_data_container.turn_instructions); + storage::serialization::write(writer, turn_data_container.lane_data_ids); + storage::serialization::write(writer, turn_data_container.travel_modes); + storage::serialization::write(writer, turn_data_container.entry_class_ids); + storage::serialization::write(writer, turn_data_container.pre_turn_bearings); + storage::serialization::write(writer, turn_data_container.post_turn_bearings); +} +} +} +} + +#endif diff --git a/include/extractor/turn_data_container.hpp b/include/extractor/turn_data_container.hpp index 139d3c210..5dbb3bf4c 100644 --- a/include/extractor/turn_data_container.hpp +++ b/include/extractor/turn_data_container.hpp @@ -6,7 +6,7 @@ #include "storage/shared_memory_ownership.hpp" #include "util/guidance/turn_bearing.hpp" -#include "util/shared_memory_vector_wrapper.hpp" +#include "util/vector_view.hpp" #include "util/typedefs.hpp" @@ -43,7 +43,7 @@ namespace detail { template class TurnDataContainerImpl { - template using Vector = typename util::ShM::vector; + template using Vector = util::ViewOrVector; public: TurnDataContainerImpl() = default; diff --git a/include/partition/cell_storage.hpp b/include/partition/cell_storage.hpp index 0c49ddbfd..c36e779ab 100644 --- a/include/partition/cell_storage.hpp +++ b/include/partition/cell_storage.hpp @@ -6,7 +6,7 @@ #include "util/assert.hpp" #include "util/for_each_range.hpp" #include "util/log.hpp" -#include "util/shared_memory_vector_wrapper.hpp" +#include "util/vector_view.hpp" #include "util/typedefs.hpp" #include "storage/io.hpp" @@ -74,7 +74,7 @@ template class CellStorageImpl }; private: - template using Vector = typename util::ShM::vector; + template using Vector = util::ViewOrVector; // Implementation of the cell view. We need a template parameter here // because we need to derive a read-only and read-write view from this. diff --git a/include/partition/multi_level_graph.hpp b/include/partition/multi_level_graph.hpp index 622a83fad..3e6146f0f 100644 --- a/include/partition/multi_level_graph.hpp +++ b/include/partition/multi_level_graph.hpp @@ -6,6 +6,7 @@ #include "storage/shared_memory_ownership.hpp" #include "util/static_graph.hpp" +#include "util/vector_view.hpp" #include @@ -41,7 +42,7 @@ class MultiLevelGraph : public util::StaticGraph { private: using SuperT = util::StaticGraph; - template using Vector = typename util::ShM::vector; + template using Vector = util::ViewOrVector; public: // We limit each node to have 255 edges diff --git a/include/partition/multi_level_partition.hpp b/include/partition/multi_level_partition.hpp index 361f5d9f4..87e67b354 100644 --- a/include/partition/multi_level_partition.hpp +++ b/include/partition/multi_level_partition.hpp @@ -4,7 +4,7 @@ #include "util/exception.hpp" #include "util/for_each_pair.hpp" #include "util/msb.hpp" -#include "util/shared_memory_vector_wrapper.hpp" +#include "util/vector_view.hpp" #include "util/typedefs.hpp" #include "storage/io.hpp" @@ -56,7 +56,7 @@ template class MultiLevelPartitionImpl final static const constexpr std::uint8_t MAX_NUM_LEVEL = 16; static const constexpr std::uint8_t NUM_PARTITION_BITS = sizeof(PartitionID) * CHAR_BIT; - template using Vector = typename util::ShM::vector; + template using Vector = util::ViewOrVector; public: // Contains all data necessary to describe the level hierarchy diff --git a/include/partition/serialization.hpp b/include/partition/serialization.hpp index acf419b5d..11220a534 100644 --- a/include/partition/serialization.hpp +++ b/include/partition/serialization.hpp @@ -7,6 +7,7 @@ #include "partition/multi_level_partition.hpp" #include "storage/io.hpp" +#include "storage/serialization.hpp" #include "storage/shared_memory_ownership.hpp" namespace osrm @@ -19,26 +20,26 @@ namespace serialization template inline void read(storage::io::FileReader &reader, MultiLevelGraph &graph) { - reader.DeserializeVector(graph.node_array); - reader.DeserializeVector(graph.edge_array); - reader.DeserializeVector(graph.node_to_edge_offset); + storage::serialization::read(reader, graph.node_array); + storage::serialization::read(reader, graph.edge_array); + storage::serialization::read(reader, graph.node_to_edge_offset); } template inline void write(storage::io::FileWriter &writer, const MultiLevelGraph &graph) { - writer.SerializeVector(graph.node_array); - writer.SerializeVector(graph.edge_array); - writer.SerializeVector(graph.node_to_edge_offset); + storage::serialization::write(writer, graph.node_array); + storage::serialization::write(writer, graph.edge_array); + storage::serialization::write(writer, graph.node_to_edge_offset); } template inline void read(storage::io::FileReader &reader, detail::MultiLevelPartitionImpl &mlp) { reader.ReadInto(*mlp.level_data); - reader.DeserializeVector(mlp.partition); - reader.DeserializeVector(mlp.cell_to_children); + storage::serialization::read(reader, mlp.partition); + storage::serialization::read(reader, mlp.cell_to_children); } template @@ -46,29 +47,29 @@ inline void write(storage::io::FileWriter &writer, const detail::MultiLevelPartitionImpl &mlp) { writer.WriteOne(*mlp.level_data); - writer.SerializeVector(mlp.partition); - writer.SerializeVector(mlp.cell_to_children); + storage::serialization::write(writer, mlp.partition); + storage::serialization::write(writer, mlp.cell_to_children); } template inline void read(storage::io::FileReader &reader, detail::CellStorageImpl &storage) { - reader.DeserializeVector(storage.weights); - reader.DeserializeVector(storage.source_boundary); - reader.DeserializeVector(storage.destination_boundary); - reader.DeserializeVector(storage.cells); - reader.DeserializeVector(storage.level_to_cell_offset); + storage::serialization::read(reader, storage.weights); + storage::serialization::read(reader, storage.source_boundary); + storage::serialization::read(reader, storage.destination_boundary); + storage::serialization::read(reader, storage.cells); + storage::serialization::read(reader, storage.level_to_cell_offset); } template inline void write(storage::io::FileWriter &writer, const detail::CellStorageImpl &storage) { - writer.SerializeVector(storage.weights); - writer.SerializeVector(storage.source_boundary); - writer.SerializeVector(storage.destination_boundary); - writer.SerializeVector(storage.cells); - writer.SerializeVector(storage.level_to_cell_offset); + storage::serialization::write(writer, storage.weights); + storage::serialization::write(writer, storage.source_boundary); + storage::serialization::write(writer, storage.destination_boundary); + storage::serialization::write(writer, storage.cells); + storage::serialization::write(writer, storage.level_to_cell_offset); } } } diff --git a/include/storage/io.hpp b/include/storage/io.hpp index fd817e8d0..6a7741ea7 100644 --- a/include/storage/io.hpp +++ b/include/storage/io.hpp @@ -128,20 +128,6 @@ class FileReader std::uint32_t ReadElementCount32() { return ReadOne(); } std::uint64_t ReadElementCount64() { return ReadOne(); } - template void DeserializeVector(VectorT &data) - { - const auto count = ReadElementCount64(); - data.resize(count); - ReadInto(data.data(), count); - } - - template std::size_t GetVectorMemorySize() - { - const auto count = ReadElementCount64(); - Skip(count); - return sizeof(count) + sizeof(T) * count; - } - template std::size_t ReadVectorSize() { const auto count = ReadElementCount64(); @@ -149,19 +135,6 @@ class FileReader return count; } - template void *DeserializeVector(void *begin, const void *end) - { - auto count = ReadElementCount64(); - auto required = reinterpret_cast(begin) + sizeof(count) + sizeof(T) * count; - if (required > end) - throw util::exception("Not enough memory "); - - *reinterpret_cast(begin) = count; - ReadInto(reinterpret_cast(reinterpret_cast(begin) + sizeof(decltype(count))), - count); - return required; - } - bool ReadAndCheckFingerprint() { auto loaded_fingerprint = ReadOne(); @@ -199,42 +172,6 @@ class FileReader input_stream.seekg(current_pos, input_stream.beg); return length; } - - std::vector ReadLines() - { - std::vector result; - std::string thisline; - try - { - while (std::getline(input_stream, thisline)) - { - result.push_back(thisline); - } - } - catch (const std::ios_base::failure &) - { - // EOF is OK here, everything else, re-throw - if (!input_stream.eof()) - throw; - } - return result; - } - - std::string ReadLine() - { - std::string thisline; - try - { - std::getline(input_stream, thisline); - } - catch (const std::ios_base::failure & /*e*/) - { - // EOF is OK here, everything else, re-throw - if (!input_stream.eof()) - throw; - } - return thisline; - } }; class FileWriter @@ -296,13 +233,6 @@ class FileWriter void WriteElementCount32(const std::uint32_t count) { WriteOne(count); } void WriteElementCount64(const std::uint64_t count) { WriteOne(count); } - template void SerializeVector(const VectorT &data) - { - const auto count = data.size(); - WriteElementCount64(count); - return WriteFrom(data.data(), count); - } - void WriteFingerprint() { const auto fingerprint = util::FingerPrint::GetValid(); diff --git a/include/storage/serialization.hpp b/include/storage/serialization.hpp new file mode 100644 index 000000000..0a99cf084 --- /dev/null +++ b/include/storage/serialization.hpp @@ -0,0 +1,47 @@ +#ifndef OSRM_STORAGE_SERIALIZATION_HPP +#define OSRM_STORAGE_SERIALIZATION_HPP + +#include "util/vector_view.hpp" + +#include "storage/io.hpp" + +namespace osrm +{ +namespace storage +{ +namespace serialization +{ + +template void read(io::FileReader &reader, std::vector &data) +{ + const auto count = reader.ReadElementCount64(); + data.resize(count); + reader.ReadInto(data.data(), count); +} + +template void write(io::FileWriter &writer, const std::vector &data) +{ + const auto count = data.size(); + writer.WriteElementCount64(count); + return writer.WriteFrom(data.data(), count); +} + +template void read(io::FileReader &reader, util::vector_view &data) +{ + const auto count = reader.ReadElementCount64(); + data.resize(count); + reader.ReadInto(data.data(), count); +} + +template void write(io::FileWriter &writer, const util::vector_view &data) +{ + const auto count = data.size(); + writer.WriteElementCount64(count); + return writer.WriteFrom(data.data(), count); +} + +} +} +} + +#endif diff --git a/include/util/name_table.hpp b/include/util/name_table.hpp index 06aa8cb5e..fdf5d07ec 100644 --- a/include/util/name_table.hpp +++ b/include/util/name_table.hpp @@ -2,7 +2,6 @@ #define OSRM_UTIL_NAME_TABLE_HPP #include "util/indexed_data.hpp" -#include "util/shared_memory_vector_wrapper.hpp" #include "util/string_view.hpp" #include "util/typedefs.hpp" diff --git a/include/util/packed_vector.hpp b/include/util/packed_vector.hpp index 126190eb3..97ac685f8 100644 --- a/include/util/packed_vector.hpp +++ b/include/util/packed_vector.hpp @@ -1,7 +1,7 @@ #ifndef PACKED_VECTOR_HPP #define PACKED_VECTOR_HPP -#include "util/shared_memory_vector_wrapper.hpp" +#include "util/vector_view.hpp" #include "util/typedefs.hpp" #include "storage/io.hpp" @@ -171,7 +171,7 @@ template class PackedVector const detail::PackedVector &vec); private: - typename util::ShM::vector vec; + util::ViewOrVector vec; std::uint64_t num_elements = 0; diff --git a/include/util/range_table.hpp b/include/util/range_table.hpp index 190686746..318d6c86f 100644 --- a/include/util/range_table.hpp +++ b/include/util/range_table.hpp @@ -4,7 +4,7 @@ #include "storage/io.hpp" #include "storage/shared_memory_ownership.hpp" #include "util/integer_range.hpp" -#include "util/shared_memory_vector_wrapper.hpp" +#include "util/vector_view.hpp" #include #include @@ -41,8 +41,8 @@ template class RangeTable { public: using BlockT = std::array; - using BlockContainerT = typename ShM::vector; - using OffsetContainerT = typename ShM::vector; + using BlockContainerT = util::ViewOrVector; + using OffsetContainerT = util::ViewOrVector; using RangeT = range; friend std::ostream &operator<<<>(std::ostream &out, const RangeTable &table); diff --git a/include/util/serialization.hpp b/include/util/serialization.hpp index 952699cd7..bff5291fa 100644 --- a/include/util/serialization.hpp +++ b/include/util/serialization.hpp @@ -4,7 +4,9 @@ #include "util/packed_vector.hpp" #include "util/static_graph.hpp" #include "util/dynamic_graph.hpp" + #include "storage/io.hpp" +#include "storage/serialization.hpp" namespace osrm { @@ -17,7 +19,7 @@ inline void read(storage::io::FileReader &reader, detail::PackedVector &vec) { vec.num_elements =reader.ReadOne(); - reader.DeserializeVector(vec.vec); + storage::serialization::read(reader, vec.vec); } template @@ -25,30 +27,30 @@ inline void write(storage::io::FileWriter &writer, const detail::PackedVector &vec) { writer.WriteOne(vec.num_elements); - writer.SerializeVector(vec.vec); + storage::serialization::write(writer, vec.vec); } template inline void read(storage::io::FileReader &reader, StaticGraph &graph) { - reader.DeserializeVector(graph.node_array); - reader.DeserializeVector(graph.edge_array); + storage::serialization::read(reader, graph.node_array); + storage::serialization::read(reader, graph.edge_array); } template inline void write(storage::io::FileWriter &writer, const StaticGraph &graph) { - writer.SerializeVector(graph.node_array); - writer.SerializeVector(graph.edge_array); + storage::serialization::write(writer, graph.node_array); + storage::serialization::write(writer, graph.edge_array); } template inline void read(storage::io::FileReader &reader, DynamicGraph &graph) { - reader.DeserializeVector(graph.node_array); + storage::serialization::read(reader, graph.node_array); auto num_edges = reader.ReadElementCount64(); graph.edge_list.resize(num_edges); for (auto index : irange(0, num_edges)) @@ -63,7 +65,7 @@ template inline void write(storage::io::FileWriter &writer, const DynamicGraph &graph) { - writer.SerializeVector(graph.node_array); + storage::serialization::write(writer, graph.node_array); writer.WriteElementCount64(graph.number_of_edges); for (auto index : irange(0, graph.number_of_edges)) { diff --git a/include/util/static_graph.hpp b/include/util/static_graph.hpp index 047952cc6..4ce3907bc 100644 --- a/include/util/static_graph.hpp +++ b/include/util/static_graph.hpp @@ -4,7 +4,7 @@ #include "util/graph_traits.hpp" #include "util/integer_range.hpp" #include "util/percent.hpp" -#include "util/shared_memory_vector_wrapper.hpp" +#include "util/vector_view.hpp" #include "util/typedefs.hpp" #include "storage/shared_memory_ownership.hpp" @@ -122,7 +122,7 @@ EntryT edgeToEntry(const OtherEdge &from, std::false_type) template class StaticGraph { - template using Vector = typename util::ShM::vector; + template using Vector = util::ViewOrVector; public: using InputEdge = static_graph_details::SortableEdgeWithData; diff --git a/include/util/static_rtree.hpp b/include/util/static_rtree.hpp index ca055662c..555f38024 100644 --- a/include/util/static_rtree.hpp +++ b/include/util/static_rtree.hpp @@ -9,7 +9,7 @@ #include "util/hilbert_value.hpp" #include "util/integer_range.hpp" #include "util/rectangle.hpp" -#include "util/shared_memory_vector_wrapper.hpp" +#include "util/vector_view.hpp" #include "util/typedefs.hpp" #include "util/web_mercator.hpp" @@ -58,7 +58,7 @@ template class StaticRTree { - template using Vector = typename ShM::vector; + template using Vector = ViewOrVector; public: using Rectangle = RectangleInt2D; @@ -160,7 +160,7 @@ class StaticRTree boost::iostreams::mapped_file_source m_leaves_region; // read-only view of leaves - typename ShM::vector m_leaves; + util::vector_view m_leaves; public: StaticRTree(const StaticRTree &) = delete; diff --git a/include/util/shared_memory_vector_wrapper.hpp b/include/util/vector_view.hpp similarity index 78% rename from include/util/shared_memory_vector_wrapper.hpp rename to include/util/vector_view.hpp index 3c361aa63..452471fef 100644 --- a/include/util/shared_memory_vector_wrapper.hpp +++ b/include/util/vector_view.hpp @@ -1,5 +1,5 @@ -#ifndef SHARED_MEMORY_VECTOR_WRAPPER_HPP -#define SHARED_MEMORY_VECTOR_WRAPPER_HPP +#ifndef UTIL_VECTOR_VIEW_HPP +#define UTIL_VECTOR_VIEW_HPP #include "util/exception.hpp" #include "util/log.hpp" @@ -25,11 +25,13 @@ namespace util { template -class ShMemIterator - : public boost::iterator_facade, DataT, boost::random_access_traversal_tag> +class VectorViewIterator : public boost::iterator_facade, + DataT, + boost::random_access_traversal_tag> { - typedef boost::iterator_facade, DataT, boost::random_access_traversal_tag> - base_t; + typedef boost:: + iterator_facade, DataT, boost::random_access_traversal_tag> + base_t; public: typedef typename base_t::value_type value_type; @@ -37,16 +39,16 @@ class ShMemIterator typedef typename base_t::reference reference; typedef std::random_access_iterator_tag iterator_category; - explicit ShMemIterator() : m_value(nullptr) {} - explicit ShMemIterator(DataT *x) : m_value(x) {} + explicit VectorViewIterator() : m_value(nullptr) {} + explicit VectorViewIterator(DataT *x) : m_value(x) {} private: void increment() { ++m_value; } void decrement() { --m_value; } void advance(difference_type offset) { m_value += offset; } - bool equal(const ShMemIterator &other) const { return m_value == other.m_value; } + bool equal(const VectorViewIterator &other) const { return m_value == other.m_value; } reference dereference() const { return *m_value; } - difference_type distance_to(const ShMemIterator &other) const + difference_type distance_to(const VectorViewIterator &other) const { return other.m_value - m_value; } @@ -63,8 +65,8 @@ template class vector_view public: using value_type = DataT; - using iterator = ShMemIterator; - using const_iterator = ShMemIterator; + using iterator = VectorViewIterator; + using const_iterator = VectorViewIterator; using reverse_iterator = boost::reverse_iterator; vector_view() : m_ptr(nullptr), m_size(0) {} @@ -88,7 +90,9 @@ template class vector_view { if (m_size != size) { - throw util::exception("Invalid resize " + std::to_string(size) + " on immutable vector view of size " + std::to_string(m_size) + "."); + throw util::exception("Invalid resize " + std::to_string(size) + + " on immutable vector view of size " + std::to_string(m_size) + + "."); } } @@ -189,12 +193,10 @@ template void swap(vector_view &lhs, vector_view std::swap(lhs.m_size, rhs.m_size); } -template struct ShM -{ - using vector = typename std::conditional, - std::vector>::type; -}; +template +using ViewOrVector = typename std::conditional, + std::vector>::type; } } diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp index 167146c5b..d4069f938 100644 --- a/src/contractor/contractor.cpp +++ b/src/contractor/contractor.cpp @@ -1,6 +1,6 @@ #include "contractor/contractor.hpp" -#include "contractor/files.hpp" #include "contractor/crc32_processor.hpp" +#include "contractor/files.hpp" #include "contractor/graph_contractor.hpp" #include "contractor/graph_contractor_adaptors.hpp" @@ -47,9 +47,9 @@ int Contractor::Run() util::Log() << "Reading node weights."; std::vector node_weights; { - storage::io::FileReader node_file(config.node_file_path, + storage::io::FileReader reader(config.node_file_path, storage::io::FileReader::VerifyFingerprint); - node_file.DeserializeVector(node_weights); + storage::serialization::read(reader, node_weights); } util::Log() << "Done reading node weights."; @@ -115,10 +115,10 @@ void Contractor::WriteNodeLevels(std::vector &&in_node_levels) const { std::vector node_levels(std::move(in_node_levels)); - storage::io::FileWriter node_level_file(config.level_output_path, - storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter writer(config.level_output_path, + storage::io::FileWriter::HasNoFingerprint); - node_level_file.SerializeVector(node_levels); + storage::serialization::write(writer, node_levels); } void Contractor::WriteCoreNodeMarker(std::vector &&in_is_core_node) const @@ -138,9 +138,8 @@ void Contractor::WriteCoreNodeMarker(std::vector &&in_is_core_node) const core_marker_output_file.WriteFrom(unpacked_bool_flags.data(), count); } -void -Contractor::WriteContractedGraph(unsigned max_node_id, - util::DeallocatingVector contracted_edge_list) +void Contractor::WriteContractedGraph(unsigned max_node_id, + util::DeallocatingVector contracted_edge_list) { // Sorting contracted edges in a way that the static query graph can read some in in-place. tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end()); @@ -150,7 +149,7 @@ Contractor::WriteContractedGraph(unsigned max_node_id, RangebasedCRC32 crc32_calculator; const unsigned checksum = crc32_calculator(contracted_edge_list); - QueryGraph query_graph {max_node_id+1, contracted_edge_list}; + QueryGraph query_graph{max_node_id + 1, contracted_edge_list}; files::writeGraph(config.graph_output_path, checksum, query_graph); } diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index f5260472a..b123bfadf 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -48,8 +48,7 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory( std::vector &turn_lane_offsets, std::vector &turn_lane_masks, guidance::LaneDescriptionMap &lane_description_map) - : m_max_edge_id(0), m_coordinates(coordinates), - m_osm_node_ids(osm_node_ids), + : m_max_edge_id(0), m_coordinates(coordinates), m_osm_node_ids(osm_node_ids), m_node_based_graph(std::move(node_based_graph)), m_restriction_map(std::move(restriction_map)), m_barrier_nodes(barrier_nodes), m_traffic_lights(traffic_lights), m_compressed_edge_container(compressed_edge_container), @@ -540,16 +539,14 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( const bool isTrivial = m_compressed_edge_container.IsTrivial(incoming_edge); const auto &from_node = - isTrivial - ? m_osm_node_ids[node_along_road_entering] - : m_osm_node_ids[m_compressed_edge_container.GetLastEdgeSourceID( - incoming_edge)]; + isTrivial ? m_osm_node_ids[node_along_road_entering] + : m_osm_node_ids[m_compressed_edge_container.GetLastEdgeSourceID( + incoming_edge)]; const auto &via_node = m_osm_node_ids[m_compressed_edge_container.GetLastEdgeTargetID( incoming_edge)]; const auto &to_node = - m_osm_node_ids[m_compressed_edge_container.GetFirstEdgeTargetID( - turn.eid)]; + m_osm_node_ids[m_compressed_edge_container.GetFirstEdgeTargetID(turn.eid)]; lookup::TurnIndexBlock turn_index_block = {from_node, via_node, to_node}; @@ -562,32 +559,32 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( // write weight penalties per turn BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size()); { - storage::io::FileWriter turn_weight_penalties_file( - turn_weight_penalties_filename, storage::io::FileWriter::HasNoFingerprint); - turn_weight_penalties_file.SerializeVector(turn_weight_penalties); + storage::io::FileWriter writer(turn_weight_penalties_filename, + storage::io::FileWriter::HasNoFingerprint); + storage::serialization::write(writer, turn_weight_penalties); } { - storage::io::FileWriter turn_duration_penalties_file( - turn_duration_penalties_filename, storage::io::FileWriter::HasNoFingerprint); - turn_duration_penalties_file.SerializeVector(turn_duration_penalties); + storage::io::FileWriter writer(turn_duration_penalties_filename, + storage::io::FileWriter::HasNoFingerprint); + storage::serialization::write(writer, turn_duration_penalties); } util::Log() << "Created " << entry_class_hash.size() << " entry classes and " << bearing_class_hash.size() << " Bearing Classes"; util::Log() << "Writing Turn Lane Data to File..."; + { + storage::io::FileWriter writer(turn_lane_data_filename, + storage::io::FileWriter::HasNoFingerprint); - storage::io::FileWriter turn_lane_data_file(turn_lane_data_filename, - storage::io::FileWriter::HasNoFingerprint); - - std::vector lane_data(lane_data_map.size()); - // extract lane data sorted by ID - for (auto itr : lane_data_map) - lane_data[itr.second] = itr.first; - - turn_lane_data_file.SerializeVector(lane_data); + std::vector lane_data(lane_data_map.size()); + // extract lane data sorted by ID + for (auto itr : lane_data_map) + lane_data[itr.second] = itr.first; + storage::serialization::write(writer, lane_data); + } util::Log() << "done."; files::writeTurnData(turn_data_filename, turn_data_container); diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 2d03e81b9..4f3033c86 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -281,11 +281,11 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) util::Log() << "Saving edge-based node weights to file."; TIMER_START(timer_write_node_weights); - - storage::io::FileWriter edge_file(config.edge_based_node_weights_output_path, - storage::io::FileWriter::GenerateFingerprint); - edge_file.SerializeVector(edge_based_node_weights); - + { + storage::io::FileWriter writer(config.edge_based_node_weights_output_path, + storage::io::FileWriter::GenerateFingerprint); + storage::serialization::write(writer, edge_based_node_weights); + } TIMER_STOP(timer_write_node_weights); util::Log() << "Done writing. (" << TIMER_SEC(timer_write_node_weights) << ")"; @@ -299,7 +299,8 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) TIMER_STOP(rtree); util::Log() << "Writing node map ..."; - files::writeNodes(config.node_output_path, coordinates, osm_node_ids); + files::writeNodes( + config.node_output_path, coordinates, osm_node_ids); WriteEdgeBasedGraph(config.edge_graph_output_path, max_edge_id, edge_based_edge_list); @@ -509,9 +510,8 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment, }; compressed_node_based_graph_writing = std::async(std::launch::async, [&] { - WriteCompressedNodeBasedGraph(config.compressed_node_based_graph_output_path, - *node_based_graph, - coordinates); + WriteCompressedNodeBasedGraph( + config.compressed_node_based_graph_output_path, *node_based_graph, coordinates); }); WriteTurnLaneData(config.turn_lane_descriptions_file_name); @@ -612,11 +612,11 @@ void Extractor::WriteIntersectionClassificationData( const std::vector &bearing_classes, const std::vector &entry_classes) const { - storage::io::FileWriter file(output_file_name, storage::io::FileWriter::GenerateFingerprint); + storage::io::FileWriter writer(output_file_name, storage::io::FileWriter::GenerateFingerprint); util::Log() << "Writing Intersection Classification Data"; TIMER_START(write_edges); - file.SerializeVector(node_based_intersection_classes); + storage::serialization::write(writer, node_based_intersection_classes); // create range table for vectors: std::vector bearing_counts; @@ -630,17 +630,17 @@ void Extractor::WriteIntersectionClassificationData( } util::RangeTable<> bearing_class_range_table(bearing_counts); - bearing_class_range_table.Write(file); + bearing_class_range_table.Write(writer); - file.WriteOne(total_bearings); + writer.WriteOne(total_bearings); for (const auto &bearing_class : bearing_classes) { const auto &bearings = bearing_class.getAvailableBearings(); - file.WriteFrom(bearings.data(), bearings.size()); + writer.WriteFrom(bearings.data(), bearings.size()); } - file.SerializeVector(entry_classes); + storage::serialization::write(writer, entry_classes); TIMER_STOP(write_edges); util::Log() << "ok, after " << TIMER_SEC(write_edges) << "s for " @@ -659,9 +659,9 @@ void Extractor::WriteTurnLaneData(const std::string &turn_lane_file) const util::Log() << "Writing turn lane masks..."; TIMER_START(turn_lane_timer); - storage::io::FileWriter file(turn_lane_file, storage::io::FileWriter::HasNoFingerprint); - file.SerializeVector(turn_lane_offsets); - file.SerializeVector(turn_lane_masks); + storage::io::FileWriter writer(turn_lane_file, storage::io::FileWriter::HasNoFingerprint); + storage::serialization::write(writer, turn_lane_offsets); + storage::serialization::write(writer, turn_lane_masks); TIMER_STOP(turn_lane_timer); util::Log() << "done (" << TIMER_SEC(turn_lane_timer) << ")"; diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 6c14f9558..e2482361b 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -34,10 +34,10 @@ #include "util/log.hpp" #include "util/packed_vector.hpp" #include "util/range_table.hpp" -#include "util/shared_memory_vector_wrapper.hpp" #include "util/static_graph.hpp" #include "util/static_rtree.hpp" #include "util/typedefs.hpp" +#include "util/vector_view.hpp" #ifdef __linux__ #include @@ -63,7 +63,7 @@ namespace storage { using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf; -using RTreeNode = util:: StaticRTree::TreeNode; +using RTreeNode = util::StaticRTree::TreeNode; using QueryGraph = util::StaticGraph; using EdgeBasedGraph = util::StaticGraph; @@ -361,7 +361,7 @@ void Storage::PopulateLayout(DataLayout &layout) io::FileReader::VerifyFingerprint); std::vector bearing_class_id_table; - intersection_file.DeserializeVector(bearing_class_id_table); + serialization::read(intersection_file, bearing_class_id_table); layout.SetBlockSize(DataLayout::BEARING_CLASSID, bearing_class_id_table.size()); @@ -386,7 +386,7 @@ void Storage::PopulateLayout(DataLayout &layout) layout.SetBlockSize(DataLayout::BEARING_VALUES, num_bearings); std::vector entry_class_table; - intersection_file.DeserializeVector(entry_class_table); + serialization::read(intersection_file, entry_class_table); layout.SetBlockSize(DataLayout::ENTRY_CLASS, entry_class_table.size()); @@ -690,11 +690,13 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) layout.GetBlockPtr(memory_ptr, DataLayout::COORDINATE_LIST); const auto osmnodeid_ptr = layout.GetBlockPtr(memory_ptr, DataLayout::OSM_NODE_ID_LIST); - util::vector_view coordinates(coordinates_ptr, layout.num_entries[DataLayout::COORDINATE_LIST]); + util::vector_view coordinates( + coordinates_ptr, layout.num_entries[DataLayout::COORDINATE_LIST]); util::PackedVectorView osm_node_ids; osm_node_ids.reset(osmnodeid_ptr, layout.num_entries[DataLayout::OSM_NODE_ID_LIST]); - extractor::files::readNodes(config.nodes_data_path, coordinates, osm_node_ids); + extractor::files::readNodes( + config.nodes_data_path, coordinates, osm_node_ids); } // load turn weight penalties @@ -790,7 +792,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) io::FileReader::VerifyFingerprint); std::vector bearing_class_id_table; - intersection_file.DeserializeVector(bearing_class_id_table); + serialization::read(intersection_file, bearing_class_id_table); const auto bearing_blocks = intersection_file.ReadElementCount32(); intersection_file.Skip(1); // sum_lengths @@ -808,7 +810,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) intersection_file.ReadInto(bearing_class_table.data(), num_bearings); std::vector entry_class_table; - intersection_file.DeserializeVector(entry_class_table); + serialization::read(intersection_file, entry_class_table); // load intersection classes if (!bearing_class_id_table.empty()) diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index 6d80e86df..bbdc85213 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -6,7 +6,6 @@ #include "extractor/edge_based_graph_factory.hpp" #include "extractor/files.hpp" #include "extractor/node_based_edge.hpp" -#include "extractor/seralization.hpp" #include "storage/io.hpp" @@ -460,14 +459,14 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e const auto load_turn_weight_penalties = [&] { using storage::io::FileReader; - FileReader file(config.turn_weight_penalties_path, FileReader::HasNoFingerprint); - file.DeserializeVector(turn_weight_penalties); + FileReader reader(config.turn_weight_penalties_path, FileReader::HasNoFingerprint); + storage::serialization::read(reader, turn_weight_penalties); }; const auto load_turn_duration_penalties = [&] { using storage::io::FileReader; - FileReader file(config.turn_duration_penalties_path, FileReader::HasNoFingerprint); - file.DeserializeVector(turn_duration_penalties); + FileReader reader(config.turn_duration_penalties_path, FileReader::HasNoFingerprint); + storage::serialization::read(reader, turn_duration_penalties); }; const auto load_profile_properties = [&] { @@ -648,8 +647,8 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e if (update_turn_penalties) { const auto save_penalties = [](const auto &filename, const auto &data) -> void { - storage::io::FileWriter file(filename, storage::io::FileWriter::HasNoFingerprint); - file.SerializeVector(data); + storage::io::FileWriter writer(filename, storage::io::FileWriter::HasNoFingerprint); + storage::serialization::write(writer, data); }; tbb::parallel_invoke( diff --git a/unit_tests/util/io.cpp b/unit_tests/util/io.cpp index 3a95ed31b..ac94da34b 100644 --- a/unit_tests/util/io.cpp +++ b/unit_tests/util/io.cpp @@ -1,4 +1,5 @@ #include "storage/io.hpp" +#include "storage/serialization.hpp" #include "util/exception.hpp" #include "util/fingerprint.hpp" #include "util/typedefs.hpp" @@ -29,12 +30,12 @@ BOOST_AUTO_TEST_CASE(io_data) { osrm::storage::io::FileWriter outfile(IO_TMP_FILE, osrm::storage::io::FileWriter::GenerateFingerprint); - outfile.SerializeVector(data_in); + osrm::storage::serialization::write(outfile, data_in); } osrm::storage::io::FileReader infile(IO_TMP_FILE, osrm::storage::io::FileReader::VerifyFingerprint); - infile.DeserializeVector(data_out); + osrm::storage::serialization::read(infile, data_out); BOOST_REQUIRE_EQUAL(data_in.size(), data_out.size()); BOOST_CHECK_EQUAL_COLLECTIONS(data_out.begin(), data_out.end(), data_in.begin(), data_in.end()); @@ -65,7 +66,7 @@ BOOST_AUTO_TEST_CASE(file_too_small) { osrm::storage::io::FileWriter outfile( IO_TOO_SMALL_FILE, osrm::storage::io::FileWriter::GenerateFingerprint); - outfile.SerializeVector(v); + osrm::storage::serialization::write(outfile, v); } std::fstream f(IO_TOO_SMALL_FILE); @@ -79,7 +80,7 @@ BOOST_AUTO_TEST_CASE(file_too_small) osrm::storage::io::FileReader infile(IO_TOO_SMALL_FILE, osrm::storage::io::FileReader::VerifyFingerprint); std::vector buffer; - infile.DeserializeVector(buffer); + osrm::storage::serialization::read(infile, buffer); BOOST_REQUIRE_MESSAGE(false, "Should not get here"); } catch (const osrm::util::exception &e) @@ -101,7 +102,7 @@ BOOST_AUTO_TEST_CASE(io_corrupt_fingerprint) osrm::storage::io::FileWriter::HasNoFingerprint); outfile.WriteOne(0xDEADBEEFCAFEFACE); - outfile.SerializeVector(v); + osrm::storage::serialization::write(outfile, v); } try @@ -130,7 +131,7 @@ BOOST_AUTO_TEST_CASE(io_incompatible_fingerprint) const auto fingerprint = osrm::util::FingerPrint::GetValid(); outfile.WriteOne(fingerprint); - outfile.SerializeVector(v); + osrm::storage::serialization::write(outfile, v); } std::fstream f(IO_INCOMPATIBLE_FINGERPRINT_FILE);