Work on relation extract refactoring to support recursive relations (superrelations)

2017-09-27 19:05:05 +03:00
parent 7851de9af8
commit c42e247d87
6 changed files with 292 additions and 123 deletions
@@ -386,6 +386,10 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
                                             turn_lane_map,
                                             scripting_environment.GetProfileProperties());

+    // get list of supported relation types
+    auto relation_types = scripting_environment.GetRelations();
+    std::sort(relation_types.begin(), relation_types.end());
+
    ExtractionRelationContainer relations;
    std::vector<std::string> restrictions = scripting_environment.GetRestrictions();
    // setup restriction parser
@@ -445,7 +449,6 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
                                                  relations,
                                                  parsed_buffer.resulting_nodes,
                                                  parsed_buffer.resulting_ways,
-                                                  parsed_buffer.resulting_relations,
                                                  parsed_buffer.resulting_restrictions);
            return parsed_buffer;
        });
@@ -453,6 +456,7 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
    // Parsed nodes and ways handler
    unsigned number_of_nodes = 0;
    unsigned number_of_ways = 0;
+    unsigned number_of_restrictions = 0;
    tbb::filter_t<ParsedBuffer, void> buffer_storage(
        tbb::filter::serial_in_order, [&](const ParsedBuffer &parsed_buffer) {

@@ -467,29 +471,57 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
            {
                extractor_callbacks->ProcessWay(result.first, result.second);
            }
-        });
-
-    // Parsed relations handler
-    unsigned number_of_relations = 0;
-    tbb::filter_t<ParsedBuffer, void> buffer_storage_relation(
-        tbb::filter::serial_in_order, [&](const ParsedBuffer &parsed_buffer) {
-
-            number_of_relations += parsed_buffer.resulting_relations.size();
-            for (const auto &result : parsed_buffer.resulting_relations)
-            {
-                /// TODO: add restriction processing
-                if (result.second.is_restriction)
-                    continue;
-
-                relations.AddRelation(result.second);
-            }

+            number_of_restrictions += parsed_buffer.resulting_restrictions.size();
            for (const auto &result : parsed_buffer.resulting_restrictions)
            {
                extractor_callbacks->ProcessRestriction(result);
            }
        });

+
+
+    tbb::filter_t<SharedBuffer, std::shared_ptr<ExtractionRelationContainer>> buffer_relation_cache(
+        tbb::filter::parallel, [&](const SharedBuffer buffer) {
+            if (!buffer)
+                return std::shared_ptr<ExtractionRelationContainer>{};
+
+            auto relations = std::make_shared<ExtractionRelationContainer>();
+            for (auto entity = buffer->cbegin(), end = buffer->cend(); entity != end; ++entity)
+            {
+                if (entity->type() != osmium::item_type::relation)
+                    continue;
+
+                const auto & rel = static_cast<const osmium::Relation &>(*entity);
+
+                const char * rel_type = rel.get_value_by_key("type");
+                if (!rel_type || !std::binary_search(relation_types.begin(), relation_types.end(), std::string(rel_type)))
+                    continue;
+
+                ExtractionRelation extracted_rel({rel.id(), osmium::item_type::relation});
+                for (auto const & t : rel.tags())
+                    extracted_rel.attributes.emplace_back(std::make_pair(t.key(), t.value()));
+
+                for (auto const & m : rel.members())
+                {
+                    ExtractionRelation::OsmIDTyped const mid(m.ref(), m.type());
+                    extracted_rel.AddMember(mid, m.role());
+                    relations->AddRelationMember(extracted_rel.id, mid);
+                }
+
+                relations->AddRelation(std::move(extracted_rel));
+            };
+            return relations;
+        });
+
+    unsigned number_of_relations = 0;
+    tbb::filter_t<std::shared_ptr<ExtractionRelationContainer>, void> buffer_storage_relation(
+        tbb::filter::serial_in_order, [&](const std::shared_ptr<ExtractionRelationContainer> parsed_relations) {
+
+            number_of_relations += parsed_relations->GetRelationsNum();
+            relations.Merge(std::move(*parsed_relations));
+        });
+
    // Parse OSM elements with parallel transformer
    // Number of pipeline tokens that yielded the best speedup was about 1.5 * num_cores
    const auto num_threads = tbb::task_scheduler_init::default_num_threads() * 1.5;
@@ -500,13 +532,13 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
        util::Log() << "Parse relations ...";
        osmium::io::Reader reader(input_file, osmium::osm_entity_bits::relation, read_meta);
        tbb::parallel_pipeline(
-            num_threads, buffer_reader(reader) & buffer_transformer & buffer_storage_relation);
+            num_threads, buffer_reader(reader) & buffer_relation_cache & buffer_storage_relation);
    }

    { // Nodes and ways reading pipeline
        util::Log() << "Parse ways and nodes ...";
        osmium::io::Reader reader(
-            input_file, osmium::osm_entity_bits::node | osmium::osm_entity_bits::way, read_meta);
+            input_file, osmium::osm_entity_bits::node | osmium::osm_entity_bits::way | osmium::osm_entity_bits::relation, read_meta);

        const auto pipeline =
            scripting_environment.HasLocationDependentData() && config.use_locations_cache
@@ -519,7 +551,8 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
    util::Log() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds";

    util::Log() << "Raw input contains " << number_of_nodes << " nodes, " << number_of_ways
-                << " ways, and " << number_of_relations << " relations";
+                << " ways, and " << number_of_relations << " relations, " << number_of_restrictions
+                << " restrictions";

    extractor_callbacks.reset();

@@ -323,29 +323,6 @@ void Sol2ScriptingEnvironment::InitContext(LuaScriptingContext &context)
            return boost::apply_visitor(to_lua_object(context.state), value);
        });

-    context.state.new_usertype<osmium::RelationMember>(
-        "RelationMember",
-        "role",
-        &osmium::RelationMember::role,
-        "type",
-        &osmium::RelationMember::type,
-        "id",
-        [](const osmium::RelationMember &member) -> osmium::object_id_type {
-            return member.ref();
-        });
-
-    context.state.new_usertype<osmium::Relation>(
-        "Relation",
-        "get_value_by_key",
-        &get_value_by_key<osmium::Relation>,
-        "id",
-        &osmium::Relation::id,
-        "version",
-        &osmium::Relation::version,
-        "members",
-        [](const osmium::Relation &rel) -> const osmium::RelationMemberList & {
-            return rel.members();
-        });

    context.state.new_usertype<osmium::Node>("Node",
                                             "location",
@@ -440,17 +417,59 @@ void Sol2ScriptingEnvironment::InitContext(LuaScriptingContext &context)
        sol::property([](const ExtractionWay &way) { return way.is_left_hand_driving; },
                      [](ExtractionWay &way, bool flag) { way.is_left_hand_driving = flag; }));

+    auto getTypedRefBySol = [](const sol::object & obj) -> ExtractionRelation::OsmIDTyped
+    {
+        if (obj.is<osmium::Way>())
+        {
+            osmium::Way * way = obj.as<osmium::Way*>();
+            return { way->id(), osmium::item_type::way };
+        }
+
+        if (obj.is<osmium::Relation>())
+        {
+            osmium::Relation * rel = obj.as<osmium::Relation*>();
+            return { rel->id(), osmium::item_type::relation };
+        }
+
+        if (obj.is<osmium::Node>())
+        {
+            osmium::Node * node = obj.as<osmium::Node*>();
+            return { node->id(), osmium::item_type::node };
+        }
+
+        return ExtractionRelation::OsmIDTyped(0, osmium::item_type::undefined);
+    };
+
+    context.state.new_usertype<ExtractionRelation::OsmIDTyped>(
+        "OsmIDTyped",
+        "id",
+        &ExtractionRelation::OsmIDTyped::GetID,
+        "type",
+        &ExtractionRelation::OsmIDTyped::GetType);
+
    context.state.new_usertype<ExtractionRelation>(
        "ExtractionRelation",
-        sol::meta_function::new_index,
-        [](ExtractionRelation &rel, const osmium::RelationMember &member)
-            -> ExtractionRelation::AttributesMap & { return rel.GetMember(member); },
-        sol::meta_function::index,
-        [](ExtractionRelation &rel, const osmium::RelationMember &member)
-            -> ExtractionRelation::AttributesMap & { return rel.GetMember(member); },
-        "restriction",
-        sol::property([](const ExtractionRelation &rel) { return rel.is_restriction; },
-                      [](ExtractionRelation &rel, bool flag) { rel.is_restriction = flag; }));
+        "get_value_by_key",
+        [](ExtractionRelation &rel, const char * key) -> const char * { return rel.GetAttr(key); },
+        "get_role",
+        [&getTypedRefBySol](ExtractionRelation &rel, const sol::object & obj) -> const char *
+        {
+            return rel.GetRole(getTypedRefBySol(obj));
+        });
+
+    context.state.new_usertype<ExtractionRelationContainer>(
+        "ExtractionRelationContainer",
+        "get_relations",
+        [&getTypedRefBySol](ExtractionRelationContainer &cont, const sol::object & obj)
+            -> const ExtractionRelationContainer::RelationIDList &
+        {
+            return cont.GetRelations(getTypedRefBySol(obj));
+        },
+        "relation",
+        [](ExtractionRelationContainer &cont, const ExtractionRelation::OsmIDTyped & rel_id) -> const ExtractionRelation &
+        {
+            return cont.GetRelationData(rel_id);
+        });

    context.state.new_usertype<ExtractionSegment>("ExtractionSegment",
                                                  "source",
@@ -633,14 +652,6 @@ void Sol2ScriptingEnvironment::InitContext(LuaScriptingContext &context)
    switch (context.api_version)
    {
    case 3:
-    {
-        initV2Context();
-        context.relation_function = function_table.value()["process_relation"];
-
-        context.has_relation_function = context.relation_function.valid();
-        break;
-    }
-
    case 2:
    {
        initV2Context();
@@ -719,12 +730,10 @@ void Sol2ScriptingEnvironment::ProcessElements(
    const ExtractionRelationContainer &relations,
    std::vector<std::pair<const osmium::Node &, ExtractionNode>> &resulting_nodes,
    std::vector<std::pair<const osmium::Way &, ExtractionWay>> &resulting_ways,
-    std::vector<std::pair<const osmium::Relation &, ExtractionRelation>> &resulting_relations,
    std::vector<InputConditionalTurnRestriction> &resulting_restrictions)
 {
    ExtractionNode result_node;
    ExtractionWay result_way;
-    ExtractionRelation result_relation;
    auto &local_context = this->GetSol2Context();

    for (auto entity = buffer.cbegin(), end = buffer.cend(); entity != end; ++entity)
@@ -738,8 +747,7 @@ void Sol2ScriptingEnvironment::ProcessElements(
            if (local_context.has_node_function &&
                (!node.tags().empty() || local_context.properties.call_tagless_node_function))
            {
-                const auto &id = ExtractionRelation::OsmIDTyped(node.id(), osmium::item_type::node);
-                local_context.ProcessNode(node, result_node, relations.Get(id));
+                local_context.ProcessNode(node, result_node, relations);
            }
            resulting_nodes.push_back({node, std::move(result_node)});
        }
@@ -750,8 +758,7 @@ void Sol2ScriptingEnvironment::ProcessElements(
            result_way.clear();
            if (local_context.has_way_function)
            {
-                const auto &id = ExtractionRelation::OsmIDTyped(way.id(), osmium::item_type::way);
-                local_context.ProcessWay(way, result_way, relations.Get(id));
+                local_context.ProcessWay(way, result_way, relations);
            }
            resulting_ways.push_back({way, std::move(result_way)});
        }
@@ -763,16 +770,6 @@ void Sol2ScriptingEnvironment::ProcessElements(
            {
                resulting_restrictions.push_back(*result_res);
            }
-
-            if (local_context.api_version > 2)
-            {
-                result_relation.clear();
-                if (local_context.has_relation_function)
-                {
-                    local_context.ProcessRelation(relation, result_relation);
-                }
-                resulting_relations.push_back({relation, std::move(result_relation)});
-            }
        }
        break;
        default:
@@ -901,6 +898,18 @@ std::vector<std::string> Sol2ScriptingEnvironment::GetRestrictions()
    }
 }

+std::vector<std::string> Sol2ScriptingEnvironment::GetRelations()
+{
+    auto &context = GetSol2Context();
+    switch (context.api_version)
+    {
+    case 3:
+        return Sol2ScriptingEnvironment::GetStringListFromTable("relation_types");
+    default:
+        return {};
+    }
+}
+
 void Sol2ScriptingEnvironment::ProcessTurn(ExtractionTurn &turn)
 {
    auto &context = GetSol2Context();
@@ -992,7 +1001,7 @@ void Sol2ScriptingEnvironment::ProcessSegment(ExtractionSegment &segment)

 void LuaScriptingContext::ProcessNode(const osmium::Node &node,
                                      ExtractionNode &result,
-                                      const ExtractionRelationContainer::RelationList &relations)
+                                      const ExtractionRelationContainer &relations)
 {
    BOOST_ASSERT(state.lua_state() != nullptr);

@@ -1013,7 +1022,7 @@ void LuaScriptingContext::ProcessNode(const osmium::Node &node,

 void LuaScriptingContext::ProcessWay(const osmium::Way &way,
                                     ExtractionWay &result,
-                                     const ExtractionRelationContainer::RelationList &relations)
+                                     const ExtractionRelationContainer &relations)
 {
    BOOST_ASSERT(state.lua_state() != nullptr);

@@ -1033,14 +1042,5 @@ void LuaScriptingContext::ProcessWay(const osmium::Way &way,
    }
 }

-void LuaScriptingContext::ProcessRelation(const osmium::Relation &relation,
-                                          ExtractionRelation &result)
-{
-    BOOST_ASSERT(state.lua_state() != nullptr);
-    BOOST_ASSERT(api_version > 2);
-
-    relation_function(profile_table, relation, result);
-}
-
 } // namespace extractor
 } // namespace osrm