Refactor compressed geometry in own abstraction with read/write

include/extractor/compressed_edge_container.hpp

@@ -1,6 +1,8 @@
 #ifndef GEOMETRY_COMPRESSOR_HPP_
 #define GEOMETRY_COMPRESSOR_HPP_
 
+#include "extractor/segment_data_container.hpp"
+
 #include "util/typedefs.hpp"
 
 #include <unordered_map>
@@ -48,7 +50,6 @@ class CompressedEdgeContainer
     bool HasZippedEntryForForwardID(const EdgeID edge_id) const;
     bool HasZippedEntryForReverseID(const EdgeID edge_id) const;
     void PrintStatistics() const;
-    void SerializeInternalVector(const std::string &path) const;
     unsigned GetPositionForID(const EdgeID edge_id) const;
     unsigned GetZippedPositionForForwardID(const EdgeID edge_id) const;
     unsigned GetZippedPositionForReverseID(const EdgeID edge_id) const;
@@ -58,21 +59,19 @@ class CompressedEdgeContainer
     NodeID GetLastEdgeTargetID(const EdgeID edge_id) const;
     NodeID GetLastEdgeSourceID(const EdgeID edge_id) const;
 
+    // Invalidates the internal storage
+    SegmentDataContainer ToSegmentData();
+
   private:
     int free_list_maximum = 0;
 
     void IncreaseFreeList();
     std::vector<OnewayEdgeBucket> m_compressed_oneway_geometries;
-    std::vector<unsigned> m_compressed_geometry_index;
-    std::vector<NodeID> m_compressed_geometry_nodes;
-    std::vector<EdgeWeight> m_compressed_geometry_fwd_weights;
-    std::vector<EdgeWeight> m_compressed_geometry_rev_weights;
-    std::vector<EdgeWeight> m_compressed_geometry_fwd_durations;
-    std::vector<EdgeWeight> m_compressed_geometry_rev_durations;
     std::vector<unsigned> m_free_list;
     std::unordered_map<EdgeID, unsigned> m_edge_id_to_list_index_map;
     std::unordered_map<EdgeID, unsigned> m_forward_edge_id_to_zipped_index_map;
     std::unordered_map<EdgeID, unsigned> m_reverse_edge_id_to_zipped_index_map;
+    SegmentDataContainer segment_data;
 };
 }
 }

include/extractor/io.hpp

@@ -0,0 +1,62 @@
+#ifndef OSRM_EXTRACTOR_IO_HPP
+#define OSRM_EXTRACTOR_IO_HPP
+
+#include "extractor/segment_data_container.hpp"
+
+#include "storage/io.hpp"
+
+namespace osrm
+{
+namespace extractor
+{
+namespace io
+{
+
+template <>
+void read(const boost::filesystem::path &path, SegmentDataContainer &segment_data)
+{
+    const auto fingerprint = storage::io::FileReader::HasNoFingerprint;
+    storage::io::FileReader reader{path, fingerprint};
+
+    auto num_indices = reader.ReadElementCount32();
+    segment_data.index.resize(num_indices);
+    reader.ReadInto(segment_data.index.data(), num_indices);
+
+    auto num_entries = reader.ReadElementCount32();
+    segment_data.nodes.resize(num_entries);
+    segment_data.fwd_weights.resize(num_entries);
+    segment_data.rev_weights.resize(num_entries);
+    segment_data.fwd_durations.resize(num_entries);
+    segment_data.rev_durations.resize(num_entries);
+
+    reader.ReadInto(segment_data.nodes.data(), segment_data.nodes.size());
+    reader.ReadInto(segment_data.fwd_weights.data(), segment_data.fwd_weights.size());
+    reader.ReadInto(segment_data.rev_weights.data(), segment_data.rev_weights.size());
+    reader.ReadInto(segment_data.fwd_durations.data(), segment_data.fwd_durations.size());
+    reader.ReadInto(segment_data.rev_durations.data(), segment_data.rev_durations.size());
+}
+
+template <> void write(const boost::filesystem::path &path, const SegmentDataContainer &segment_data)
+{
+    const auto fingerprint = storage::io::FileWriter::HasNoFingerprint;
+    storage::io::FileWriter writer{path, fingerprint};
+
+    // FIXME this _should_ just be size and the senitel below need to be removed
+    writer.WriteElementCount32(segment_data.index.size() + 1);
+    writer.WriteFrom(segment_data.index.data(), segment_data.index.size());
+    // FIMXE remove unnecessary senitel
+    writer.WriteElementCount32(segment_data.nodes.size());
+
+    writer.WriteElementCount32(segment_data.nodes.size());
+    writer.WriteFrom(segment_data.nodes.data(), segment_data.nodes.size());
+    writer.WriteFrom(segment_data.fwd_weights.data(), segment_data.fwd_weights.size());
+    writer.WriteFrom(segment_data.rev_weights.data(), segment_data.rev_weights.size());
+    writer.WriteFrom(segment_data.fwd_durations.data(), segment_data.fwd_durations.size());
+    writer.WriteFrom(segment_data.rev_durations.data(), segment_data.rev_durations.size());
+}
+
+}
+}
+}
+
+#endif

include/extractor/segment_data_container.hpp

@@ -0,0 +1,126 @@
+#ifndef OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_
+#define OSRM_EXTRACTOR_SEGMENT_DATA_CONTAINER_HPP_
+
+#include "util/shared_memory_vector_wrapper.hpp"
+#include "util/typedefs.hpp"
+
+#include <boost/range/adaptor/reversed.hpp>
+#include <boost/range/iterator_range.hpp>
+#include <boost/filesystem/path.hpp>
+
+#include <unordered_map>
+
+#include <string>
+#include <vector>
+
+namespace osrm
+{
+namespace extractor
+{
+
+class CompressedEdgeContainer;
+
+namespace detail
+{
+template <bool UseShareMemory> class SegmentDataContainerImpl;
+}
+
+namespace io
+{
+template <bool UseShareMemory>
+inline void read(const boost::filesystem::path &path, detail::SegmentDataContainerImpl<UseShareMemory> &segment_data);
+template <bool UseShareMemory>
+inline void write(const boost::filesystem::path &path, const detail::SegmentDataContainerImpl<UseShareMemory> &segment_data);
+}
+
+namespace detail
+{
+template <bool UseShareMemory> class SegmentDataContainerImpl
+{
+    template <typename T> using Vector = typename util::ShM<T, UseShareMemory>::vector;
+
+    friend CompressedEdgeContainer;
+
+  public:
+    using SegmentID = std::uint32_t;
+
+    SegmentDataContainerImpl() = default;
+
+    SegmentDataContainerImpl(Vector<std::uint32_t> index_,
+                             Vector<NodeID> nodes_,
+                             Vector<EdgeWeight> fwd_weights_,
+                             Vector<EdgeWeight> rev_weights_,
+                             Vector<EdgeWeight> fwd_durations_,
+                             Vector<EdgeWeight> rev_durations_)
+        : index(std::move(index_)), nodes(std::move(nodes_)), fwd_weights(std::move(fwd_weights_)),
+          rev_weights(std::move(rev_weights_)), fwd_durations(std::move(fwd_durations_)),
+          rev_durations(std::move(rev_durations_))
+    {
+    }
+
+    auto GetForwardGeometry(const SegmentID id) const
+    {
+        const auto begin = nodes.begin() + index.at(id);
+        const auto end = nodes.begin() + index.at(id + 1);
+
+        return boost::make_iterator_range(begin, end);
+    }
+
+    auto GetReverseGeometry(const SegmentID id) const
+    {
+        return boost::adaptors::reverse(GetForwardGeometry(id));
+    }
+
+    auto GetForwardDurations(const SegmentID id) const
+    {
+        const auto begin = fwd_durations.begin() + index.at(id) + 1;
+        const auto end = fwd_durations.begin() + index.at(id + 1);
+
+        return boost::make_iterator_range(begin, end);
+    }
+
+    auto GetReverseDurations(const SegmentID id) const
+    {
+        const auto begin = rev_durations.begin() + index.at(id);
+        const auto end = rev_durations.begin() + index.at(id + 1) - 1;
+
+        return boost::adaptors::reverse(boost::make_iterator_range(begin, end));
+    }
+
+    auto GetForwardWeights(const SegmentID id) const
+    {
+        const auto begin = fwd_weights.begin() + index.at(id) + 1;
+        const auto end = fwd_weights.begin() + index.at(id + 1);
+
+        return boost::make_iterator_range(begin, end);
+    }
+
+    auto GetReverseWeights(const SegmentID id) const
+    {
+        const auto begin = rev_weights.begin() + index.at(id);
+        const auto end = rev_weights.begin() + index.at(id + 1) - 1;
+
+        return boost::adaptors::reverse(boost::make_iterator_range(begin, end));
+    }
+
+    friend void io::read<UseShareMemory>(const boost::filesystem::path &path,
+                                         detail::SegmentDataContainerImpl<UseShareMemory> &segment_data);
+    friend void io::write<UseShareMemory>(const boost::filesystem::path &path,
+                                          const detail::SegmentDataContainerImpl<UseShareMemory> &segment_data);
+
+  private:
+    Vector<std::uint32_t> index;
+    Vector<NodeID> nodes;
+    Vector<EdgeWeight> fwd_weights;
+    Vector<EdgeWeight> rev_weights;
+    Vector<EdgeWeight> fwd_durations;
+    Vector<EdgeWeight> rev_durations;
+};
+}
+
+using SegmentDataView = detail::SegmentDataContainerImpl<true>;
+using SegmentDataContainer = detail::SegmentDataContainerImpl<false>;
+}
+}
+
+#endif

src/extractor/compressed_edge_container.cpp

@@ -61,7 +61,7 @@ unsigned CompressedEdgeContainer::GetZippedPositionForForwardID(const EdgeID edg
 {
     auto map_iterator = m_forward_edge_id_to_zipped_index_map.find(edge_id);
     BOOST_ASSERT(map_iterator != m_forward_edge_id_to_zipped_index_map.end());
-    BOOST_ASSERT(map_iterator->second < m_compressed_geometry_nodes.size());
+    BOOST_ASSERT(map_iterator->second < segment_data.nodes.size());
     return map_iterator->second;
 }
 
@@ -69,48 +69,10 @@ unsigned CompressedEdgeContainer::GetZippedPositionForReverseID(const EdgeID edg
 {
     auto map_iterator = m_reverse_edge_id_to_zipped_index_map.find(edge_id);
     BOOST_ASSERT(map_iterator != m_reverse_edge_id_to_zipped_index_map.end());
-    BOOST_ASSERT(map_iterator->second < m_compressed_geometry_nodes.size());
+    BOOST_ASSERT(map_iterator->second < segment_data.nodes.size());
     return map_iterator->second;
 }
 
-void CompressedEdgeContainer::SerializeInternalVector(const std::string &path) const
-{
-    boost::filesystem::fstream geometry_out_stream(path, std::ios::binary | std::ios::out);
-    const unsigned compressed_geometry_indices = m_compressed_geometry_index.size() + 1;
-    const unsigned compressed_geometries = m_compressed_geometry_nodes.size();
-    BOOST_ASSERT(std::numeric_limits<unsigned>::max() != compressed_geometry_indices);
-    geometry_out_stream.write((char *)&compressed_geometry_indices, sizeof(unsigned));
-
-    geometry_out_stream.write((char *)(m_compressed_geometry_index.data()),
-                              sizeof(unsigned) * m_compressed_geometry_index.size());
-
-    // sentinel element
-    geometry_out_stream.write((char *)&(compressed_geometries), sizeof(unsigned));
-
-    // number of geometry entries to follow
-    geometry_out_stream.write((char *)&(compressed_geometries), sizeof(unsigned));
-
-    // write compressed geometry node id's
-    geometry_out_stream.write((char *)(m_compressed_geometry_nodes.data()),
-                              sizeof(NodeID) * m_compressed_geometry_nodes.size());
-
-    // write compressed geometry forward weights
-    geometry_out_stream.write((char *)(m_compressed_geometry_fwd_weights.data()),
-                              sizeof(EdgeWeight) * m_compressed_geometry_fwd_weights.size());
-
-    // write compressed geometry reverse weights
-    geometry_out_stream.write((char *)(m_compressed_geometry_rev_weights.data()),
-                              sizeof(EdgeWeight) * m_compressed_geometry_rev_weights.size());
-
-    // write compressed geometry forward durations
-    geometry_out_stream.write((char *)(m_compressed_geometry_fwd_durations.data()),
-                              sizeof(EdgeWeight) * m_compressed_geometry_fwd_durations.size());
-
-    // write compressed geometry reverse durations
-    geometry_out_stream.write((char *)(m_compressed_geometry_rev_durations.data()),
-                              sizeof(EdgeWeight) * m_compressed_geometry_rev_durations.size());
-}
-
 // Adds info for a compressed edge to the container.   edge_id_2
 // has been removed from the graph, so we have to save These edges/nodes
 // have already been trimmed from the graph, this function just stores
@@ -252,12 +214,12 @@ void CompressedEdgeContainer::AddUncompressedEdge(const EdgeID edge_id,
 
 void CompressedEdgeContainer::InitializeBothwayVector()
 {
-    m_compressed_geometry_index.reserve(m_compressed_oneway_geometries.size());
+    segment_data.index.reserve(m_compressed_oneway_geometries.size());
-    m_compressed_geometry_nodes.reserve(m_compressed_oneway_geometries.size());
+    segment_data.nodes.reserve(m_compressed_oneway_geometries.size());
-    m_compressed_geometry_fwd_weights.reserve(m_compressed_oneway_geometries.size());
+    segment_data.fwd_weights.reserve(m_compressed_oneway_geometries.size());
-    m_compressed_geometry_rev_weights.reserve(m_compressed_oneway_geometries.size());
+    segment_data.rev_weights.reserve(m_compressed_oneway_geometries.size());
-    m_compressed_geometry_fwd_durations.reserve(m_compressed_oneway_geometries.size());
+    segment_data.fwd_durations.reserve(m_compressed_oneway_geometries.size());
-    m_compressed_geometry_rev_durations.reserve(m_compressed_oneway_geometries.size());
+    segment_data.rev_durations.reserve(m_compressed_oneway_geometries.size());
 }
 
 unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID r_edge_id)
@@ -267,19 +229,19 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
 
     BOOST_ASSERT(forward_bucket.size() == reverse_bucket.size());
 
-    const unsigned zipped_geometry_id = m_compressed_geometry_index.size();
+    const unsigned zipped_geometry_id = segment_data.index.size();
     m_forward_edge_id_to_zipped_index_map[f_edge_id] = zipped_geometry_id;
     m_reverse_edge_id_to_zipped_index_map[r_edge_id] = zipped_geometry_id;
 
-    m_compressed_geometry_index.emplace_back(m_compressed_geometry_nodes.size());
+    segment_data.index.emplace_back(segment_data.nodes.size());
 
     const auto &first_node = reverse_bucket.back();
 
-    m_compressed_geometry_nodes.emplace_back(first_node.node_id);
+    segment_data.nodes.emplace_back(first_node.node_id);
-    m_compressed_geometry_fwd_weights.emplace_back(INVALID_EDGE_WEIGHT);
+    segment_data.fwd_weights.emplace_back(INVALID_EDGE_WEIGHT);
-    m_compressed_geometry_rev_weights.emplace_back(first_node.weight);
+    segment_data.rev_weights.emplace_back(first_node.weight);
-    m_compressed_geometry_fwd_durations.emplace_back(INVALID_EDGE_WEIGHT);
+    segment_data.fwd_durations.emplace_back(INVALID_EDGE_WEIGHT);
-    m_compressed_geometry_rev_durations.emplace_back(first_node.duration);
+    segment_data.rev_durations.emplace_back(first_node.duration);
 
     for (std::size_t i = 0; i < forward_bucket.size() - 1; ++i)
     {
@@ -288,20 +250,20 @@ unsigned CompressedEdgeContainer::ZipEdges(const EdgeID f_edge_id, const EdgeID
 
         BOOST_ASSERT(fwd_node.node_id == rev_node.node_id);
 
-        m_compressed_geometry_nodes.emplace_back(fwd_node.node_id);
+        segment_data.nodes.emplace_back(fwd_node.node_id);
-        m_compressed_geometry_fwd_weights.emplace_back(fwd_node.weight);
+        segment_data.fwd_weights.emplace_back(fwd_node.weight);
-        m_compressed_geometry_rev_weights.emplace_back(rev_node.weight);
+        segment_data.rev_weights.emplace_back(rev_node.weight);
-        m_compressed_geometry_fwd_durations.emplace_back(fwd_node.duration);
+        segment_data.fwd_durations.emplace_back(fwd_node.duration);
-        m_compressed_geometry_rev_durations.emplace_back(rev_node.duration);
+        segment_data.rev_durations.emplace_back(rev_node.duration);
     }
 
     const auto &last_node = forward_bucket.back();
 
-    m_compressed_geometry_nodes.emplace_back(last_node.node_id);
+    segment_data.nodes.emplace_back(last_node.node_id);
-    m_compressed_geometry_fwd_weights.emplace_back(last_node.weight);
+    segment_data.fwd_weights.emplace_back(last_node.weight);
-    m_compressed_geometry_rev_weights.emplace_back(INVALID_EDGE_WEIGHT);
+    segment_data.rev_weights.emplace_back(INVALID_EDGE_WEIGHT);
-    m_compressed_geometry_fwd_durations.emplace_back(last_node.duration);
+    segment_data.fwd_durations.emplace_back(last_node.duration);
-    m_compressed_geometry_rev_durations.emplace_back(INVALID_EDGE_WEIGHT);
+    segment_data.rev_durations.emplace_back(INVALID_EDGE_WEIGHT);
 
     return zipped_geometry_id;
 }
@@ -364,5 +326,11 @@ NodeID CompressedEdgeContainer::GetLastEdgeSourceID(const EdgeID edge_id) const
     BOOST_ASSERT(bucket.size() >= 2);
     return bucket[bucket.size() - 2].node_id;
 }
+
+SegmentDataContainer CompressedEdgeContainer::ToSegmentData()
+{
+    return std::move(segment_data);
+}
+
 }
 }

src/extractor/extractor.cpp

@@ -5,10 +5,11 @@
 #include "extractor/extraction_node.hpp"
 #include "extractor/extraction_way.hpp"
 #include "extractor/extractor_callbacks.hpp"
+#include "extractor/io.hpp"
+#include "extractor/raster_source.hpp"
 #include "extractor/restriction_parser.hpp"
 #include "extractor/scripting_environment.hpp"
 
-#include "extractor/raster_source.hpp"
 #include "storage/io.hpp"
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
@@ -513,7 +514,7 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
     });
 
     WriteTurnLaneData(config.turn_lane_descriptions_file_name);
-    compressed_edge_container.SerializeInternalVector(config.geometry_output_path);
+    io::write(config.geometry_output_path, compressed_edge_container.ToSegmentData());
 
     edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list);
     edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list);