Implements Mapping for NodeBasedGraph -> EdgeBasedgraph Translation

include/extractor/edge_based_graph_factory.hpp

@@ -7,18 +7,16 @@
 #include "extractor/edge_based_edge.hpp"
 #include "extractor/edge_based_node.hpp"
 #include "extractor/extraction_turn.hpp"
+#include "extractor/guidance/turn_analysis.hpp"
+#include "extractor/guidance/turn_instruction.hpp"
+#include "extractor/guidance/turn_lane_types.hpp"
 #include "extractor/original_edge_data.hpp"
 #include "extractor/profile_properties.hpp"
 #include "extractor/query_node.hpp"
 #include "extractor/restriction_map.hpp"
-
+#include "util/deallocating_vector.hpp"
-#include "extractor/guidance/turn_analysis.hpp"
-#include "extractor/guidance/turn_instruction.hpp"
-#include "extractor/guidance/turn_lane_types.hpp"
 #include "util/guidance/bearing_class.hpp"
 #include "util/guidance/entry_class.hpp"
-
-#include "util/deallocating_vector.hpp"
 #include "util/name_table.hpp"
 #include "util/node_based_graph.hpp"
 #include "util/typedefs.hpp"
@@ -35,6 +33,7 @@
 #include <vector>
 
 #include <boost/filesystem/fstream.hpp>
+#include <boost/optional.hpp>
 
 namespace osrm
 {
@@ -86,7 +85,9 @@ struct TurnIndexBlock
 #pragma pack(pop)
 static_assert(std::is_trivial<TurnIndexBlock>::value, "TurnIndexBlock is not trivial");
 static_assert(sizeof(TurnIndexBlock) == 24, "TurnIndexBlock is not packed correctly");
-}
+} // ns lookup
+
+struct NodeBasedGraphToEdgeBasedGraphMappingWriter; // fwd. decl
 
 class EdgeBasedGraphFactory
 {
@@ -113,7 +114,9 @@ class EdgeBasedGraphFactory
              const std::string &turn_weight_penalties_filename,
              const std::string &turn_duration_penalties_filename,
              const std::string &turn_penalties_index_filename,
-             const bool generate_edge_lookup);
+             const bool generate_edge_lookup,
+             const bool generate_nbg_ebg_mapping,
+             const std::string &nbg_ebg_mapping_path);
 
     // The following get access functions destroy the content in the factory
     void GetEdgeBasedEdges(util::DeallocatingVector<EdgeBasedEdge> &edges);
@@ -172,7 +175,10 @@ class EdgeBasedGraphFactory
     guidance::LaneDescriptionMap &lane_description_map;
 
     unsigned RenumberEdges();
-    void GenerateEdgeExpandedNodes();
+
+    void GenerateEdgeExpandedNodes(const bool generate_nbg_ebg_mapping,
+                                   const std::string &nbg_ebg_mapping_path);
+
     void GenerateEdgeExpandedEdges(ScriptingEnvironment &scripting_environment,
                                    const std::string &original_edge_data_filename,
                                    const std::string &turn_lane_data_filename,
@@ -182,7 +188,15 @@ class EdgeBasedGraphFactory
                                    const std::string &turn_penalties_index_filename,
                                    const bool generate_edge_lookup);
 
-    void InsertEdgeBasedNode(const NodeID u, const NodeID v);
+    // Mapping betweenn the node based graph u,v nodes and the edge based graph head,tail edge ids.
+    // Required in the osrm-partition tool to translate from a nbg partition to a ebg partition.
+    struct Mapping
+    {
+        NodeID u, v;
+        EdgeID head, tail;
+    };
+
+    boost::optional<Mapping> InsertEdgeBasedNode(const NodeID u, const NodeID v);
 
     void FlushVectorToStream(std::ofstream &edge_data_file,
                              std::vector<OriginalEdgeData> &original_edge_data_vector) const;

include/extractor/extractor_config.hpp

@@ -78,6 +78,7 @@ struct ExtractorConfig
         profile_properties_output_path = basepath + ".osrm.properties";
         intersection_class_data_output_path = basepath + ".osrm.icd";
         compressed_node_based_graph_output_path = basepath + ".osrm.cnbg";
+        nbg_ebg_graph_mapping_output_path = basepath + ".osrm.nbg_to_ebg";
     }
 
     boost::filesystem::path input_path;
@@ -110,8 +111,10 @@ struct ExtractorConfig
 
     bool use_metadata;
 
+    // Auxiliary data for osrm-partition
     bool dump_compressed_node_based_graph;
     std::string compressed_node_based_graph_output_path;
+    std::string nbg_ebg_graph_mapping_output_path;
 };
 }
 }

include/extractor/node_based_graph_to_edge_based_graph_mapping_writer.hpp

@@ -0,0 +1,63 @@
+#ifndef NODE_BASED_GRAPH_TO_EDGE_BASED_GRAPH_MAPPING_WRITER_HPP
+#define NODE_BASED_GRAPH_TO_EDGE_BASED_GRAPH_MAPPING_WRITER_HPP
+
+#include "storage/io.hpp"
+#include "util/log.hpp"
+#include "util/typedefs.hpp"
+
+#include <boost/assert.hpp>
+
+#include <cstddef>
+
+#include <string>
+
+namespace osrm
+{
+namespace extractor
+{
+
+// Writes:  | Fingerprint | #mappings | u v fwd_node bkw_node | u v fwd_node bkw_node | ..
+// - uint64: number of mappings (u, v, fwd_node bkw_node) chunks
+// - NodeID u, NodeID v, EdgeID fwd_node, EdgeID bkw_node
+
+struct NodeBasedGraphToEdgeBasedGraphMappingWriter
+{
+    NodeBasedGraphToEdgeBasedGraphMappingWriter(const std::string &path)
+        : writer{path, storage::io::FileWriter::GenerateFingerprint}, num_written{0}
+    {
+        const std::uint64_t dummy{0}; // filled in later
+        writer.WriteElementCount64(dummy);
+    }
+
+    void WriteMapping(NodeID u, NodeID v, EdgeID fwd_ebg_node, EdgeID bkw_ebg_node)
+    {
+        BOOST_ASSERT(u != SPECIAL_NODEID);
+        BOOST_ASSERT(v != SPECIAL_NODEID);
+        BOOST_ASSERT(fwd_ebg_node != SPECIAL_EDGEID || bkw_ebg_node != SPECIAL_EDGEID);
+
+        writer.WriteOne(u);
+        writer.WriteOne(v);
+        writer.WriteOne(fwd_ebg_node);
+        writer.WriteOne(bkw_ebg_node);
+
+        num_written += 1;
+    }
+
+    ~NodeBasedGraphToEdgeBasedGraphMappingWriter()
+    {
+        if (num_written != 0)
+        {
+            writer.SkipToBeginning();
+            writer.WriteOne(num_written);
+        }
+    }
+
+  private:
+    storage::io::FileWriter writer;
+    std::uint64_t num_written;
+};
+
+} // ns extractor
+} // ns osrm
+
+#endif // NODE_BASED_GRAPH_TO_EDGE_BASED_GRAPH_MAPPING_WRITER_HPP

include/partition/partition_config.hpp

@@ -32,6 +32,7 @@ struct PartitionConfig
 
         edge_based_graph_path = basepath + ".osrm.ebg";
         compressed_node_based_graph_path = basepath + ".osrm.cnbg";
+        nbg_ebg_mapping_path = basepath + ".osrm.nbg_to_ebg";
         partition_path = basepath + ".osrm.partition";
     }
 
@@ -39,6 +40,7 @@ struct PartitionConfig
     boost::filesystem::path base_path;
     boost::filesystem::path edge_based_graph_path;
     boost::filesystem::path compressed_node_based_graph_path;
+    boost::filesystem::path nbg_ebg_mapping_path;
     boost::filesystem::path partition_path;
 
     unsigned requested_num_threads;

include/storage/io.hpp

@@ -43,6 +43,7 @@ class FileReader
         VerifyFingerprint,
         HasNoFingerprint
     };
+
     FileReader(const std::string &filename, const FingerprintFlag flag)
         : FileReader(boost::filesystem::path(filename), flag)
     {
@@ -210,10 +211,6 @@ class FileReader
 
 class FileWriter
 {
-  private:
-    const boost::filesystem::path filepath;
-    boost::filesystem::ofstream output_stream;
-
   public:
     enum FingerprintFlag
     {
@@ -227,7 +224,7 @@ class FileWriter
     }
 
     FileWriter(const boost::filesystem::path &filepath_, const FingerprintFlag flag)
-        : filepath(filepath_)
+        : filepath(filepath_), fingerprint(flag)
     {
         output_stream.open(filepath, std::ios::binary);
         if (!output_stream)
@@ -278,9 +275,30 @@ class FileWriter
         const auto fingerprint = util::FingerPrint::GetValid();
         return WriteOne(fingerprint);
     }
+
+    template <typename T> void Skip(const std::size_t element_count)
+    {
+        boost::iostreams::seek(output_stream, element_count * sizeof(T), BOOST_IOS::cur);
+    }
+
+    void SkipToBeginning()
+    {
+        boost::iostreams::seek(output_stream, 0, std::ios::beg);
+
+        // If we wrote a Fingerprint, skip over it
+        if (fingerprint == FingerprintFlag::GenerateFingerprint)
+            Skip<util::FingerPrint>(1);
+
+        // Should probably return a functor for jumping back to the current pos.
+    }
+
+  private:
+    const boost::filesystem::path filepath;
+    boost::filesystem::ofstream output_stream;
+    FingerprintFlag fingerprint;
 };
-}
+} // ns io
-}
+} // ns storage
-}
+} // ns osrm
 
 #endif

src/extractor/edge_based_graph_factory.cpp

@@ -1,5 +1,10 @@
 #include "extractor/edge_based_graph_factory.hpp"
 #include "extractor/edge_based_edge.hpp"
+#include "extractor/guidance/turn_analysis.hpp"
+#include "extractor/guidance/turn_lane_handler.hpp"
+#include "extractor/node_based_graph_to_edge_based_graph_mapping_writer.hpp"
+#include "extractor/scripting_environment.hpp"
+#include "extractor/suffix_table.hpp"
 #include "util/bearing.hpp"
 #include "util/coordinate.hpp"
 #include "util/coordinate_calculation.hpp"
@@ -10,11 +15,6 @@
 #include "util/percent.hpp"
 #include "util/timing_util.hpp"
 
-#include "extractor/guidance/turn_analysis.hpp"
-#include "extractor/guidance/turn_lane_handler.hpp"
-#include "extractor/scripting_environment.hpp"
-#include "extractor/suffix_table.hpp"
-
 #include <boost/assert.hpp>
 #include <boost/numeric/conversion/cast.hpp>
 
@@ -92,7 +92,8 @@ void EdgeBasedGraphFactory::GetEdgeBasedNodeWeights(std::vector<EdgeWeight> &out
 
 EdgeID EdgeBasedGraphFactory::GetHighestEdgeID() { return m_max_edge_id; }
 
-void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeID node_v)
+boost::optional<EdgeBasedGraphFactory::Mapping>
+EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeID node_v)
 {
     // merge edges together into one EdgeBasedNode
     BOOST_ASSERT(node_u != SPECIAL_NODEID);
@@ -112,7 +113,7 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI
 
     if (forward_data.edge_id == SPECIAL_NODEID && reverse_data.edge_id == SPECIAL_NODEID)
     {
-        return;
+        return boost::none;
     }
 
     if (forward_data.edge_id != SPECIAL_NODEID && reverse_data.edge_id == SPECIAL_NODEID)
@@ -172,6 +173,8 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeI
     }
 
     BOOST_ASSERT(current_edge_source_coordinate_id == node_v);
+
+    return Mapping{node_u, node_v, forward_data.edge_id, reverse_data.edge_id};
 }
 
 void EdgeBasedGraphFactory::FlushVectorToStream(
@@ -193,7 +196,9 @@ void EdgeBasedGraphFactory::Run(ScriptingEnvironment &scripting_environment,
                                 const std::string &turn_weight_penalties_filename,
                                 const std::string &turn_duration_penalties_filename,
                                 const std::string &turn_penalties_index_filename,
-                                const bool generate_edge_lookup)
+                                const bool generate_edge_lookup,
+                                const bool generate_nbg_ebg_mapping,
+                                const std::string &nbg_ebg_mapping_path)
 {
     TIMER_START(renumber);
     m_max_edge_id = RenumberEdges() - 1;
@@ -201,7 +206,7 @@ void EdgeBasedGraphFactory::Run(ScriptingEnvironment &scripting_environment,
 
     TIMER_START(generate_nodes);
     m_edge_based_node_weights.reserve(m_max_edge_id + 1);
-    GenerateEdgeExpandedNodes();
+    GenerateEdgeExpandedNodes(generate_nbg_ebg_mapping, nbg_ebg_mapping_path);
     TIMER_STOP(generate_nodes);
 
     TIMER_START(generate_edges);
@@ -255,8 +260,16 @@ unsigned EdgeBasedGraphFactory::RenumberEdges()
 }
 
 /// Creates the nodes in the edge expanded graph from edges in the node-based graph.
-void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes()
+void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes(const bool generate_nbg_ebg_mapping,
+                                                      const std::string &nbg_ebg_mapping_path)
 {
+    // Optional writer, for writing out a mapping. Neither default ctor not boost::optional work
+    // with the underlying FileWriter, so hack around that limitation with a unique_ptr.
+    std::unique_ptr<NodeBasedGraphToEdgeBasedGraphMappingWriter> writer;
+    if (generate_nbg_ebg_mapping)
+        writer =
+            std::make_unique<NodeBasedGraphToEdgeBasedGraphMappingWriter>(nbg_ebg_mapping_path);
+
     util::Log() << "Generating edge expanded nodes ... ";
     {
         util::UnbufferedLog log;
@@ -286,15 +299,20 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes()
 
                 BOOST_ASSERT(node_u < node_v);
 
+                boost::optional<Mapping> mapping;
+
                 // if we found a non-forward edge reverse and try again
                 if (edge_data.edge_id == SPECIAL_NODEID)
                 {
-                    InsertEdgeBasedNode(node_v, node_u);
+                    mapping = InsertEdgeBasedNode(node_v, node_u);
                 }
                 else
                 {
-                    InsertEdgeBasedNode(node_u, node_v);
+                    mapping = InsertEdgeBasedNode(node_u, node_v);
                 }
+
+                if (generate_nbg_ebg_mapping)
+                    writer->WriteMapping(mapping->u, mapping->v, mapping->head, mapping->tail);
             }
         }
     }

src/extractor/extractor.cpp

@@ -504,7 +504,9 @@ Extractor::BuildEdgeExpandedGraph(ScriptingEnvironment &scripting_environment,
                                  config.turn_weight_penalties_path,
                                  config.turn_duration_penalties_path,
                                  config.turn_penalties_index_path,
-                                 config.generate_edge_lookup);
+                                 config.generate_edge_lookup,
+                                 config.dump_compressed_node_based_graph,
+                                 config.nbg_ebg_graph_mapping_output_path);
 
     // The osrm-partition tool requires the compressed node based graph with an embedding.
     //

src/partition/partitioner.cpp

@@ -10,6 +10,7 @@
 
 #include <iterator>
 #include <tuple>
+#include <unordered_map>
 #include <vector>
 
 #include <boost/assert.hpp>
@@ -39,6 +40,8 @@ struct CompressedNodeBasedGraph
         // - uint64: number of nodes and therefore also coordinates
         // - (uint32_t, uint32_t): num_edges * edges
         // - (int32_t, int32_t: num_nodes * coordinates (lon, lat)
+        //
+        // Gets written in Extractor::WriteCompressedNodeBasedGraph
 
         const auto num_edges = reader.ReadElementCount64();
         const auto num_nodes = reader.ReadElementCount64();
@@ -57,13 +60,69 @@ struct CompressedNodeBasedGraph
 CompressedNodeBasedGraph LoadCompressedNodeBasedGraph(const std::string &path)
 {
     const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-
     storage::io::FileReader reader(path, fingerprint);
 
     CompressedNodeBasedGraph graph{reader};
     return graph;
 }
 
+struct NodeBasedGraphToEdgeBasedGraphMapping
+{
+    NodeBasedGraphToEdgeBasedGraphMapping(storage::io::FileReader &reader)
+    {
+        // Reads:  | Fingerprint | #mappings | u v fwd_node bkw_node | u v fwd_node bkw_node | ..
+        // - uint64: number of mappings (u, v, fwd_node, bkw_node) chunks
+        // - NodeID u, NodeID v, EdgeID fwd_node, EdgeID bkw_node
+        //
+        // Gets written in NodeBasedGraphToEdgeBasedGraphMappingWriter
+
+        const auto num_mappings = reader.ReadElementCount64();
+
+        edge_based_node_to_node_based_nodes.reserve(num_mappings * 2);
+
+        for (std::uint64_t i{0}; i < num_mappings; ++i)
+        {
+
+            const auto u = reader.ReadOne<NodeID>();            // node based graph `from` node
+            const auto v = reader.ReadOne<NodeID>();            // node based graph `to` node
+            const auto fwd_ebg_node = reader.ReadOne<EdgeID>(); // edge based graph forward node
+            const auto bkw_ebg_node = reader.ReadOne<EdgeID>(); // edge based graph backward node
+
+            edge_based_node_to_node_based_nodes.insert({fwd_ebg_node, {u, v}});
+            edge_based_node_to_node_based_nodes.insert({bkw_ebg_node, {v, u}});
+        }
+    }
+
+    struct NodeBasedNodes
+    {
+        NodeID u, v;
+    };
+
+    NodeBasedNodes Lookup(EdgeID edge_based_node) const
+    {
+        auto it = edge_based_node_to_node_based_nodes.find(edge_based_node);
+
+        if (it != end(edge_based_node_to_node_based_nodes))
+            return it->second;
+
+        BOOST_ASSERT_MSG(false, "unable to fine edge based node, graph <-> mapping out of sync");
+        return NodeBasedNodes{SPECIAL_NODEID, SPECIAL_NODEID};
+    }
+
+  private:
+    std::unordered_map<EdgeID, NodeBasedNodes> edge_based_node_to_node_based_nodes;
+};
+
+NodeBasedGraphToEdgeBasedGraphMapping
+LoadNodeBasedGraphToEdgeBasedGraphMapping(const std::string &path)
+{
+    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
+    storage::io::FileReader reader(path, fingerprint);
+
+    NodeBasedGraphToEdgeBasedGraphMapping mapping{reader};
+    return mapping;
+}
+
 void LogStatistics(const std::string &filename, std::vector<std::uint32_t> bisection_ids)
 {
     auto compressed_node_based_graph = LoadCompressedNodeBasedGraph(filename);
@@ -85,7 +144,7 @@ void LogStatistics(const std::string &filename, std::vector<std::uint32_t> bisec
     std::cout << "Annotation took " << TIMER_SEC(annotation) << " seconds" << std::endl;
 }
 
-void LogGeojson(const std::string &filename, std::vector<std::uint32_t> bisection_ids)
+void LogGeojson(const std::string &filename, const std::vector<std::uint32_t> &bisection_ids)
 {
     // reload graph, since we destroyed the old one
     auto compressed_node_based_graph = LoadCompressedNodeBasedGraph(filename);
@@ -173,6 +232,10 @@ int Partitioner::Run(const PartitionConfig &config)
     LogStatistics(config.compressed_node_based_graph_path.string(),
                   recursive_bisection.BisectionIDs());
 
+    auto mapping = LoadNodeBasedGraphToEdgeBasedGraphMapping(config.nbg_ebg_mapping_path.string());
+
+    util::Log() << "Loaded node based graph to edge based graph mapping";
+
     return 0;
 }