Pass relation data to way and node functions

include/extractor/extraction_relation.hpp

@@ -5,6 +5,7 @@
 
 #include <string>
 #include <unordered_map>
+#include <vector>
 
 namespace osrm
 {
@@ -45,15 +46,39 @@ struct ExtractionRelation
         return values[id.Hash()];
     }
 
-//    AttributesMap & operator[] (util::OsmIDTyped id)
-//    {
-//        return values[id];
-//    }
-
     bool is_restriction;
     std::unordered_map<util::OsmIDTyped::HashType, AttributesMap> values;
 };
 
+// It contains data of all parsed relations for each node/way element
+class ExtractionRelationContainer
+{
+public:
+    using AttributesMap = ExtractionRelation::AttributesMap;
+    using RelationList = std::vector<AttributesMap>;
+
+    void AddRelation(const ExtractionRelation & rel)
+    {
+        BOOST_ASSERT(!rel.is_restriction);
+        for (auto it : rel.values)
+            data[it.first].push_back(it.second);
+    }
+
+    const RelationList & Get(const util::OsmIDTyped & id) const
+    {
+        const auto it = data.find(id.Hash());
+        if (it != data.end())
+            return it->second;
+
+        static RelationList empty;
+        return empty;
+    }
+
+private:
+    // TODO: need to store more common data
+    std::unordered_map<util::OsmIDTyped::HashType, RelationList> data;
+};
+
 } // namespace extractor
 } // namespace osrm
 

include/extractor/extractor_callbacks.hpp

@@ -15,6 +15,7 @@ namespace osmium
 {
 class Node;
 class Way;
+class Relation;
 }
 
 namespace std
@@ -44,6 +45,7 @@ namespace extractor
 class ExtractionContainers;
 struct ExtractionNode;
 struct ExtractionWay;
+struct ExtractionRelation;
 struct ProfileProperties;
 struct InputConditionalTurnRestriction;
 
@@ -88,6 +90,7 @@ class ExtractorCallbacks
     // warning: caller needs to take care of synchronization!
     void ProcessWay(const osmium::Way &current_way, const ExtractionWay &result_way);
 };
+
 }
 }
 

include/extractor/scripting_environment.hpp

@@ -34,6 +34,7 @@ namespace extractor
 {
 
 class RestrictionParser;
+class ExtractionRelationContainer;
 struct ExtractionNode;
 struct ExtractionWay;
 struct ExtractionRelation;
@@ -64,6 +65,7 @@ class ScriptingEnvironment
     virtual void
     ProcessElements(const osmium::memory::Buffer &buffer,
                     const RestrictionParser &restriction_parser,
+                    const ExtractionRelationContainer &relations,
                     std::vector<std::pair<const osmium::Node &, ExtractionNode>> &resulting_nodes,
                     std::vector<std::pair<const osmium::Way &, ExtractionWay>> &resulting_ways,
                     std::vector<std::pair<const osmium::Relation &, ExtractionRelation>> &resulting_relations,

include/extractor/scripting_environment_lua.hpp

@@ -3,6 +3,7 @@
 
 #include "extractor/raster_source.hpp"
 #include "extractor/scripting_environment.hpp"
+#include "extractor/extraction_relation.hpp"
 
 #include <tbb/enumerable_thread_specific.h>
 
@@ -19,8 +20,8 @@ namespace extractor
 
 struct LuaScriptingContext final
 {
-    void ProcessNode(const osmium::Node &, ExtractionNode &result);
+    void ProcessNode(const osmium::Node &, ExtractionNode &result, const ExtractionRelationContainer::RelationList &relations);
-    void ProcessWay(const osmium::Way &, ExtractionWay &result);
+    void ProcessWay(const osmium::Way &, ExtractionWay &result, const ExtractionRelationContainer::RelationList &relations);
     void ProcessRelation(const osmium::Relation &, ExtractionRelation &result);
 
     ProfileProperties properties;
@@ -71,6 +72,7 @@ class Sol2ScriptingEnvironment final : public ScriptingEnvironment
     void
     ProcessElements(const osmium::memory::Buffer &buffer,
                     const RestrictionParser &restriction_parser,
+                    const ExtractionRelationContainer &relations,
                     std::vector<std::pair<const osmium::Node &, ExtractionNode>> &resulting_nodes,
                     std::vector<std::pair<const osmium::Way &, ExtractionWay>> &resulting_ways,
                     std::vector<std::pair<const osmium::Relation &, ExtractionRelation>> &resulting_relations,

src/extractor/extractor.cpp

@@ -289,12 +289,12 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
 
     const osmium::io::File input_file(config.input_path.string());
     osmium::thread::Pool pool(number_of_threads);
-    osmium::io::Reader reader(
-        input_file,
-        pool,
-        (config.use_metadata ? osmium::io::read_meta::yes : osmium::io::read_meta::no));
 
-    const osmium::io::Header header = reader.header();
+    std::unique_ptr<osmium::io::Reader> reader(
+        new osmium::io::Reader(input_file, osmium::osm_entity_bits::relation,
+                               (config.use_metadata ? osmium::io::read_meta::yes : osmium::io::read_meta::no)));
+
+    osmium::io::Header header = reader->header();
 
     unsigned number_of_nodes = 0;
     unsigned number_of_ways = 0;
@@ -332,6 +332,7 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
 
     timestamp_file.WriteFrom(timestamp.c_str(), timestamp.length());
 
+    ExtractionRelationContainer relations;
     std::vector<std::string> restrictions = scripting_environment.GetRestrictions();
     // setup restriction parser
     const RestrictionParser restriction_parser(
@@ -339,8 +340,6 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
         config.parse_conditionals,
         restrictions);
 
-    std::mutex process_mutex;
-
     using SharedBuffer = std::shared_ptr<const osmium::memory::Buffer>;
     struct ParsedBuffer
     {
@@ -353,7 +352,7 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
 
     tbb::filter_t<void, SharedBuffer> buffer_reader(
         tbb::filter::serial_in_order, [&](tbb::flow_control &fc) {
-            if (auto buffer = reader.read())
+            if (auto buffer = reader->read())
             {
                 return std::make_shared<const osmium::memory::Buffer>(std::move(buffer));
             }
@@ -372,6 +371,7 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
             parsed_buffer->buffer = buffer;
             scripting_environment.ProcessElements(*buffer,
                                                   restriction_parser,
+                                                  relations,
                                                   parsed_buffer->resulting_nodes,
                                                   parsed_buffer->resulting_ways,
                                                   parsed_buffer->resulting_relations,
@@ -394,13 +394,44 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
             {
                 extractor_callbacks->ProcessWay(result.first, result.second);
             }
-            number_of_relations += parsed_buffer->resulting_restrictions.size();
+        });
+
+     tbb::filter_t<std::shared_ptr<ParsedBuffer>, void> buffer_storage_relation(
+        tbb::filter::serial_in_order, [&](const std::shared_ptr<ParsedBuffer> parsed_buffer) {
+            if (!parsed_buffer)
+                return;
+
+            number_of_relations += parsed_buffer->resulting_relations.size();
+            for (const auto &result : parsed_buffer->resulting_relations)
+            {
+                /// TODO: add restriction processing
+                if (result.second.is_restriction)
+                    continue;
+
+                relations.AddRelation(result.second);
+            }
+
             for (const auto &result : parsed_buffer->resulting_restrictions)
             {
                 extractor_callbacks->ProcessRestriction(result);
             }
         });
 
+    /* Main trick that we can use the same pipeline. It just receive relation objects
+     * from osmium. So other containers would be empty and doesn't process anything
+     */
+    util::Log() << "Parse relations ...";
+    tbb::parallel_pipeline(tbb::task_scheduler_init::default_num_threads() * 1.5,
+                           buffer_reader & buffer_transform & buffer_storage_relation);
+    reader->close();
+
+    /* At this step we just filter ways and nodes from osmium, so any relation wouldn't be
+     * processed there.
+     */
+    util::Log() << "Parse ways and nodes ...";
+    reader.reset(new osmium::io::Reader(input_file, osmium::osm_entity_bits::node | osmium::osm_entity_bits::way,
+                                        (config.use_metadata ? osmium::io::read_meta::yes : osmium::io::read_meta::no)));
+
     // Number of pipeline tokens that yielded the best speedup was about 1.5 * num_cores
     tbb::parallel_pipeline(tbb::task_scheduler_init::default_num_threads() * 1.5,
                            buffer_reader & buffer_transform & buffer_storage);

src/extractor/scripting_environment_lua.cpp

@@ -737,6 +737,7 @@ LuaScriptingContext &Sol2ScriptingEnvironment::GetSol2Context()
 void Sol2ScriptingEnvironment::ProcessElements(
     const osmium::memory::Buffer &buffer,
     const RestrictionParser &restriction_parser,
+    const ExtractionRelationContainer &relations,
     std::vector<std::pair<const osmium::Node &, ExtractionNode>> &resulting_nodes,
     std::vector<std::pair<const osmium::Way &, ExtractionWay>> &resulting_ways,
     std::vector<std::pair<const osmium::Relation &, ExtractionRelation>> &resulting_relations,
@@ -757,7 +758,9 @@ void Sol2ScriptingEnvironment::ProcessElements(
                 (!static_cast<const osmium::Node &>(*entity).tags().empty() ||
                  local_context.properties.call_tagless_node_function))
             {
-                local_context.ProcessNode(static_cast<const osmium::Node &>(*entity), result_node);
+                const osmium::Node & node = static_cast<const osmium::Node &>(*entity);
+                const util::OsmIDTyped id(node.id(), std::uint8_t(node.type()));
+                local_context.ProcessNode(node, result_node, relations.Get(id));
             }
             resulting_nodes.push_back(std::pair<const osmium::Node &, ExtractionNode>(
                 static_cast<const osmium::Node &>(*entity), std::move(result_node)));
@@ -766,7 +769,9 @@ void Sol2ScriptingEnvironment::ProcessElements(
             result_way.clear();
             if (local_context.has_way_function)
             {
-                local_context.ProcessWay(static_cast<const osmium::Way &>(*entity), result_way);
+                const osmium::Way & way = static_cast<const osmium::Way &>(*entity);
+                const util::OsmIDTyped id(way.id(), std::uint8_t(way.type()));
+                local_context.ProcessWay(way, result_way, relations.Get(id));
             }
             resulting_ways.push_back(std::pair<const osmium::Way &, ExtractionWay>(
                 static_cast<const osmium::Way &>(*entity), std::move(result_way)));
@@ -1007,15 +1012,15 @@ void Sol2ScriptingEnvironment::ProcessSegment(ExtractionSegment &segment)
     }
 }
 
-void LuaScriptingContext::ProcessNode(const osmium::Node &node, ExtractionNode &result)
+void LuaScriptingContext::ProcessNode(const osmium::Node &node, ExtractionNode &result, const ExtractionRelationContainer::RelationList &relations)
 {
     BOOST_ASSERT(state.lua_state() != nullptr);
 
     switch (api_version)
     {
     case 3:
-//        BOOST_ASSERT(false); // TODO: implement me
+        node_function(profile_table, node, result, relations);
-//        break;
+        break;
     case 2:
         node_function(profile_table, node, result);
         break;
@@ -1026,15 +1031,15 @@ void LuaScriptingContext::ProcessNode(const osmium::Node &node, ExtractionNode &
     }
 }
 
-void LuaScriptingContext::ProcessWay(const osmium::Way &way, ExtractionWay &result)
+void LuaScriptingContext::ProcessWay(const osmium::Way &way, ExtractionWay &result, const ExtractionRelationContainer::RelationList &relations)
 {
     BOOST_ASSERT(state.lua_state() != nullptr);
 
     switch (api_version)
     {
     case 3:
-//        BOOST_ASSERT(false); // TODO: implement me
+        way_function(profile_table, way, result, relations);
-//        break;
+        break;
     case 2:
         way_function(profile_table, way, result);
         break;

unit_tests/mocks/mock_scripting_environment.hpp

@@ -36,9 +36,10 @@ class MockScriptingEnvironment : public extractor::ScriptingEnvironment
 
     void ProcessElements(const osmium::memory::Buffer &,
                          const extractor::RestrictionParser &,
+                         const extractor::ExtractionRelationContainer &,
                          std::vector<std::pair<const osmium::Node &, extractor::ExtractionNode>> &,
                          std::vector<std::pair<const osmium::Way &, extractor::ExtractionWay>> &,
-                         std::vector<std::pair<const osmium::Relation &, extractor::ExtractionRelation>> &resulting_relations,
+                         std::vector<std::pair<const osmium::Relation &, extractor::ExtractionRelation>> &,
                          std::vector<extractor::InputConditionalTurnRestriction> &) override final
     {
     }