diff --git a/include/extractor/compressed_edge_container.hpp b/include/extractor/compressed_edge_container.hpp index 72c93f9dc..7916e6d59 100644 --- a/include/extractor/compressed_edge_container.hpp +++ b/include/extractor/compressed_edge_container.hpp @@ -1,6 +1,8 @@ #ifndef GEOMETRY_COMPRESSOR_HPP_ #define GEOMETRY_COMPRESSOR_HPP_ +#include "extractor/segment_data_container.hpp" + #include "util/typedefs.hpp" #include @@ -48,7 +50,6 @@ class CompressedEdgeContainer bool HasZippedEntryForForwardID(const EdgeID edge_id) const; bool HasZippedEntryForReverseID(const EdgeID edge_id) const; void PrintStatistics() const; - void SerializeInternalVector(const std::string &path) const; unsigned GetPositionForID(const EdgeID edge_id) const; unsigned GetZippedPositionForForwardID(const EdgeID edge_id) const; unsigned GetZippedPositionForReverseID(const EdgeID edge_id) const; @@ -58,21 +59,19 @@ class CompressedEdgeContainer NodeID GetLastEdgeTargetID(const EdgeID edge_id) const; NodeID GetLastEdgeSourceID(const EdgeID edge_id) const; + // Invalidates the internal storage + SegmentDataContainer ToSegmentData(); + private: int free_list_maximum = 0; void IncreaseFreeList(); std::vector m_compressed_oneway_geometries; - std::vector m_compressed_geometry_index; - std::vector m_compressed_geometry_nodes; - std::vector m_compressed_geometry_fwd_weights; - std::vector m_compressed_geometry_rev_weights; - std::vector m_compressed_geometry_fwd_durations; - std::vector m_compressed_geometry_rev_durations; std::vector m_free_list; std::unordered_map m_edge_id_to_list_index_map; std::unordered_map m_forward_edge_id_to_zipped_index_map; std::unordered_map m_reverse_edge_id_to_zipped_index_map; + SegmentDataContainer segment_data; }; } } diff --git a/include/extractor/io.hpp b/include/extractor/io.hpp new file mode 100644 index 000000000..bec3adb89 --- /dev/null +++ b/include/extractor/io.hpp @@ -0,0 +1,62 @@ +#ifndef OSRM_EXTRACTOR_IO_HPP +#define OSRM_EXTRACTOR_IO_HPP + +#include "extractor/segment_data_container.hpp" + +#include "storage/io.hpp" + +namespace osrm +{ +namespace extractor +{ +namespace io +{ + +template <> +void read(const boost::filesystem::path &path, SegmentDataContainer &segment_data) +{ + const auto fingerprint = storage::io::FileReader::HasNoFingerprint; + storage::io::FileReader reader{path, fingerprint}; + + auto num_indices = reader.ReadElementCount32(); + segment_data.index.resize(num_indices); + reader.ReadInto(segment_data.index.data(), num_indices); + + auto num_entries = reader.ReadElementCount32(); + segment_data.nodes.resize(num_entries); + segment_data.fwd_weights.resize(num_entries); + segment_data.rev_weights.resize(num_entries); + segment_data.fwd_durations.resize(num_entries); + segment_data.rev_durations.resize(num_entries); + + reader.ReadInto(segment_data.nodes.data(), segment_data.nodes.size()); + reader.ReadInto(segment_data.fwd_weights.data(), segment_data.fwd_weights.size()); + reader.ReadInto(segment_data.rev_weights.data(), segment_data.rev_weights.size()); + reader.ReadInto(segment_data.fwd_durations.data(), segment_data.fwd_durations.size()); + reader.ReadInto(segment_data.rev_durations.data(), segment_data.rev_durations.size()); +} + +template <> void write(const boost::filesystem::path &path, const SegmentDataContainer &segment_data) +{ + const auto fingerprint = storage::io::FileWriter::HasNoFingerprint; + storage::io::FileWriter writer{path, fingerprint}; + + // FIXME this _should_ just be size and the senitel below need to be removed + writer.WriteElementCount32(segment_data.index.size() + 1); + writer.WriteFrom(segment_data.index.data(), segment_data.index.size()); + // FIMXE remove unnecessary senitel + writer.WriteElementCount32(segment_data.nodes.size()); + + writer.WriteElementCount32(segment_data.nodes.size()); + writer.WriteFrom(segment_data.nodes.data(), segment_data.nodes.size()); + writer.WriteFrom(segment_data.fwd_weights.data(), segment_data.fwd_weights.size()); + writer.WriteFrom(segment_data.rev_weights.data(), segment_data.rev_weights.size()); + writer.WriteFrom(segment_data.fwd_durations.data(), segment_data.fwd_durations.size()); + writer.WriteFrom(segment_data.rev_durations.data(), segment_data.rev_durations.size()); +} + +} +} +} + +#endif diff --git a/include/extractor/segment_data_container.hpp b/include/extractor/segment_data_container.hpp new file mode 100644 index 000000000..6e860027f --- /dev/null +++ b/include/extractor/segment_data_container.hpp @@ -0,0 +1,126 @@ +#ifndef OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_ +#define OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_ + +#include "util/shared_memory_vector_wrapper.hpp" +#include "util/typedefs.hpp" + +#include +#include +#include + +#include + +#include +#include + +namespace osrm +{ +namespace extractor +{ + +class CompressedEdgeContainer; + +namespace detail +{ +template class SegmentDataContainerImpl; +} + +namespace io +{ +template +inline void read(const boost::filesystem::path &path, detail::SegmentDataContainerImpl &segment_data); +template +inline void write(const boost::filesystem::path &path, const detail::SegmentDataContainerImpl &segment_data); +} + +namespace detail +{ +template class SegmentDataContainerImpl +{ + template using Vector = typename util::ShM::vector; + + friend CompressedEdgeContainer; + + public: + using SegmentID = std::uint32_t; + + SegmentDataContainerImpl() = default; + + SegmentDataContainerImpl(Vector index_, + Vector nodes_, + Vector fwd_weights_, + Vector rev_weights_, + Vector fwd_durations_, + Vector rev_durations_) + : index(std::move(index_)), nodes(std::move(nodes_)), fwd_weights(std::move(fwd_weights_)), + rev_weights(std::move(rev_weights_)), fwd_durations(std::move(fwd_durations_)), + rev_durations(std::move(rev_durations_)) + { + } + + auto GetForwardGeometry(const SegmentID id) const + { + const auto begin = nodes.begin() + index.at(id); + const auto end = nodes.begin() + index.at(id + 1); + + return boost::make_iterator_range(begin, end); + } + + auto GetReverseGeometry(const SegmentID id) const + { + return boost::adaptors::reverse(GetForwardGeometry(id)); + } + + auto GetForwardDurations(const SegmentID id) const + { + const auto begin = fwd_durations.begin() + index.at(id) + 1; + const auto end = fwd_durations.begin() + index.at(id + 1); + + return boost::make_iterator_range(begin, end); + } + + auto GetReverseDurations(const SegmentID id) const + { + const auto begin = rev_durations.begin() + index.at(id); + const auto end = rev_durations.begin() + index.at(id + 1) - 1; + + return boost::adaptors::reverse(boost::make_iterator_range(begin, end)); + } + + auto GetForwardWeights(const SegmentID id) const + { + const auto begin = fwd_weights.begin() + index.at(id) + 1; + const auto end = fwd_weights.begin() + index.at(id + 1); + + return boost::make_iterator_range(begin, end); + } + + auto GetReverseWeights(const SegmentID id) const + { + const auto begin = rev_weights.begin() + index.at(id); + const auto end = rev_weights.begin() + index.at(id + 1) - 1; + + return boost::adaptors::reverse(boost::make_iterator_range(begin, end)); + } + + friend void io::read(const boost::filesystem::path &path, + detail::SegmentDataContainerImpl &segment_data); + friend void io::write(const boost::filesystem::path &path, + const detail::SegmentDataContainerImpl &segment_data); + + private: + Vector index; + Vector nodes; + Vector fwd_weights; + Vector rev_weights; + Vector fwd_durations; + Vector rev_durations; +}; +} + +using SegmentDataView = detail::SegmentDataContainerImpl; +using SegmentDataContainer = detail::SegmentDataContainerImpl; +} +} + +#endif diff --git a/src/extractor/compressed_edge_container.cpp b/src/extractor/compressed_edge_container.cpp index b34ab90e7..a9aee8c96 100644 --- a/src/extractor/compressed_edge_container.cpp +++ b/src/extractor/compressed_edge_container.cpp @@ -61,7 +61,7 @@ unsigned CompressedEdgeContainer::GetZippedPositionForForwardID(const EdgeID edg { auto map_iterator = m_forward_edge_id_to_zipped_index_map.find(edge_id); BOOST_ASSERT(map_iterator != m_forward_edge_id_to_zipped_index_map.end()); - BOOST_ASSERT(map_iterator->second < m_compressed_geometry_nodes.size()); + BOOST_ASSERT(map_iterator->second < segment_data.nodes.size()); return map_iterator->second; } @@ -69,48 +69,10 @@ unsigned CompressedEdgeContainer::GetZippedPositionForReverseID(const EdgeID edg { auto map_iterator = m_reverse_edge_id_to_zipped_index_map.find(edge_id); BOOST_ASSERT(map_iterator != m_reverse_edge_id_to_zipped_index_map.end()); - BOOST_ASSERT(map_iterator->second < m_compressed_geometry_nodes.size()); + BOOST_ASSERT(map_iterator->second < segment_data.nodes.size()); return map_iterator->second; } -void CompressedEdgeContainer::SerializeInternalVector(const std::string &path) const -{ - boost::filesystem::fstream geometry_out_stream(path, std::ios::binary | std::ios::out); - const unsigned compressed_geometry_indices = m_compressed_geometry_index.size() + 1; - const unsigned compressed_geometries = m_compressed_geometry_nodes.size(); - BOOST_ASSERT(std::numeric_limits::max() != compressed_geometry_indices); - geometry_out_stream.write((char *)&compressed_geometry_indices, sizeof(unsigned)); - - geometry_out_stream.write((char *)(m_compressed_geometry_index.data()), - sizeof(unsigned) * m_compressed_geometry_index.size()); - - // sentinel element - geometry_out_stream.write((char *)&(compressed_geometries), sizeof(unsigned)); - - // number of geometry entries to follow - geometry_out_stream.write((char *)&(compressed_geometries), sizeof(unsigned)); - - // write compressed geometry node id's - geometry_out_stream.write((char *)(m_compressed_geometry_nodes.data()), - sizeof(NodeID) * m_compressed_geometry_nodes.size()); - - // write compressed geometry forward weights - geometry_out_stream.write((char *)(m_compressed_geometry_fwd_weights.data()), - sizeof(EdgeWeight) * m_compressed_geometry_fwd_weights.size()); - - // write compressed geometry reverse weights - geometry_out_stream.write((char *)(m_compressed_geometry_rev_weights.data()), - sizeof(EdgeWeight) * m_compressed_geometry_rev_weights.size()); - - // write compressed geometry forward durations - geometry_out_stream.write((char *)(m_compressed_geometry_fwd_durations.data()), - sizeof(EdgeWeight) * m_compressed_geometry_fwd_durations.size()); - - // write compressed geometry reverse durations - geometry_out_stream.write((char *)(m_compressed_geometry_rev_durations.data()), - sizeof(EdgeWeight) * m_compressed_geometry_rev_durations.size()); -} - // Adds info for a compressed edge to the container. edge_id_2 // has been removed from the graph, so we have to save These edges/nodes // have already been trimmed from the graph, this function just stores @@ -252,12 +214,12 @@ void CompressedEdgeContainer::AddUncompressedEdge(const EdgeID edge_id, void CompressedEdgeContainer::InitializeBothwayVector() { - m_compressed_geometry_index.reserve(m_compressed_oneway_geometries.size()); - m_compressed_geometry_nodes.reserve(m_compressed_oneway_geometries.size()); - m_compressed_geometry_fwd_weights.reserve(m_compressed_oneway_geometries.size()); - m_compressed_geometry_rev_weights.reserve(m_compressed_oneway_geometries.size()); - m_compressed_geometry_fwd_durations.reserve(m_compressed_oneway_geometries.size()); - m_compressed_geometry_rev_durations.reserve(m_compressed_oneway_geometries.size()); + segment_data.index.reserve(m_compressed_oneway_geometries.size()); + segment_data.nodes.reserve(m_compressed_oneway_geometries.size()); + segment_data.fwd_weights.reserve(m_compressed_oneway_geometries.size()); + segment_data.rev_weights.reserve(m_compressed_oneway_geometries.size()); + segment_data.fwd_durations.reserve(m_compressed_oneway_geometries.size()); + segment_data.rev_durations.reserve(m_compressed_oneway_geometries.size()); } unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID r_edge_id) @@ -267,19 +229,19 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID BOOST_ASSERT(forward_bucket.size() == reverse_bucket.size()); - const unsigned zipped_geometry_id = m_compressed_geometry_index.size(); + const unsigned zipped_geometry_id = segment_data.index.size(); m_forward_edge_id_to_zipped_index_map[f_edge_id] = zipped_geometry_id; m_reverse_edge_id_to_zipped_index_map[r_edge_id] = zipped_geometry_id; - m_compressed_geometry_index.emplace_back(m_compressed_geometry_nodes.size()); + segment_data.index.emplace_back(segment_data.nodes.size()); const auto &first_node = reverse_bucket.back(); - m_compressed_geometry_nodes.emplace_back(first_node.node_id); - m_compressed_geometry_fwd_weights.emplace_back(INVALID_EDGE_WEIGHT); - m_compressed_geometry_rev_weights.emplace_back(first_node.weight); - m_compressed_geometry_fwd_durations.emplace_back(INVALID_EDGE_WEIGHT); - m_compressed_geometry_rev_durations.emplace_back(first_node.duration); + segment_data.nodes.emplace_back(first_node.node_id); + segment_data.fwd_weights.emplace_back(INVALID_EDGE_WEIGHT); + segment_data.rev_weights.emplace_back(first_node.weight); + segment_data.fwd_durations.emplace_back(INVALID_EDGE_WEIGHT); + segment_data.rev_durations.emplace_back(first_node.duration); for (std::size_t i = 0; i < forward_bucket.size() - 1; ++i) { @@ -288,20 +250,20 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID BOOST_ASSERT(fwd_node.node_id == rev_node.node_id); - m_compressed_geometry_nodes.emplace_back(fwd_node.node_id); - m_compressed_geometry_fwd_weights.emplace_back(fwd_node.weight); - m_compressed_geometry_rev_weights.emplace_back(rev_node.weight); - m_compressed_geometry_fwd_durations.emplace_back(fwd_node.duration); - m_compressed_geometry_rev_durations.emplace_back(rev_node.duration); + segment_data.nodes.emplace_back(fwd_node.node_id); + segment_data.fwd_weights.emplace_back(fwd_node.weight); + segment_data.rev_weights.emplace_back(rev_node.weight); + segment_data.fwd_durations.emplace_back(fwd_node.duration); + segment_data.rev_durations.emplace_back(rev_node.duration); } const auto &last_node = forward_bucket.back(); - m_compressed_geometry_nodes.emplace_back(last_node.node_id); - m_compressed_geometry_fwd_weights.emplace_back(last_node.weight); - m_compressed_geometry_rev_weights.emplace_back(INVALID_EDGE_WEIGHT); - m_compressed_geometry_fwd_durations.emplace_back(last_node.duration); - m_compressed_geometry_rev_durations.emplace_back(INVALID_EDGE_WEIGHT); + segment_data.nodes.emplace_back(last_node.node_id); + segment_data.fwd_weights.emplace_back(last_node.weight); + segment_data.rev_weights.emplace_back(INVALID_EDGE_WEIGHT); + segment_data.fwd_durations.emplace_back(last_node.duration); + segment_data.rev_durations.emplace_back(INVALID_EDGE_WEIGHT); return zipped_geometry_id; } @@ -364,5 +326,11 @@ NodeID CompressedEdgeContainer::GetLastEdgeSourceID(const EdgeID edge_id) const BOOST_ASSERT(bucket.size() >= 2); return bucket[bucket.size() - 2].node_id; } + +SegmentDataContainer CompressedEdgeContainer::ToSegmentData() +{ + return std::move(segment_data); +} + } } diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 46847f8e4..636da2470 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -5,10 +5,11 @@ #include "extractor/extraction_node.hpp" #include "extractor/extraction_way.hpp" #include "extractor/extractor_callbacks.hpp" +#include "extractor/io.hpp" +#include "extractor/raster_source.hpp" #include "extractor/restriction_parser.hpp" #include "extractor/scripting_environment.hpp" -#include "extractor/raster_source.hpp" #include "storage/io.hpp" #include "util/exception.hpp" #include "util/exception_utils.hpp" @@ -513,7 +514,7 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment, }); WriteTurnLaneData(config.turn_lane_descriptions_file_name); - compressed_edge_container.SerializeInternalVector(config.geometry_output_path); + io::write(config.geometry_output_path, compressed_edge_container.ToSegmentData()); edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list); edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list);