diff --git a/include/extractor/extraction_relation.hpp b/include/extractor/extraction_relation.hpp index 8e7a3181f..58806bc1b 100644 --- a/include/extractor/extraction_relation.hpp +++ b/include/extractor/extraction_relation.hpp @@ -1,6 +1,8 @@ #ifndef EXTRACTION_RELATION_HPP #define EXTRACTION_RELATION_HPP +#include "util/exception.hpp" + #include #include @@ -16,63 +18,197 @@ namespace extractor struct ExtractionRelation { - using AttributesMap = std::unordered_map; - using OsmIDTyped = std::pair; - - struct OsmIDTypedHash + class OsmIDTyped { - std::size_t operator()(const OsmIDTyped &id) const + public: + OsmIDTyped(osmium::object_id_type _id, osmium::item_type _type) + : id(_id), type(_type) { - return id.first ^ (static_cast(id.second) << 56); } + + std::uint64_t GetID() const { return std::uint64_t(id); } + osmium::item_type GetType() const { return type; } + + std::uint64_t Hash() const + { + return id ^ (static_cast(type) << 56); + } + + private: + osmium::object_id_type id; + osmium::item_type type; }; - ExtractionRelation() : is_restriction(false) {} + using AttributesList = std::vector>; + using MembersRolesList = std::vector>; - void clear() + explicit ExtractionRelation(const OsmIDTyped & _id) + : id(_id) { - is_restriction = false; - values.clear(); } - bool IsRestriction() const { return is_restriction; } - - AttributesMap &GetMember(const osmium::RelationMember &member) + void Clear() { - return values[OsmIDTyped(member.ref(), member.type())]; + attributes.clear(); + members_role.clear(); } - bool is_restriction; - std::unordered_map values; + const char * GetAttr(const std::string & attr) const + { + auto it = std::lower_bound( + attributes.begin(), + attributes.end(), + std::make_pair(attr, std::string())); + + if (it != attributes.end() && (*it).first == attr) + return (*it).second.c_str(); + + return nullptr; + } + + void Prepare() + { + std::sort(attributes.begin(), attributes.end()); + std::sort(members_role.begin(), members_role.end()); + } + + void AddMember(const OsmIDTyped & member_id, const char * role) + { + members_role.emplace_back(std::make_pair(member_id.Hash(), std::string(role))); + } + + const char * GetRole(const OsmIDTyped & member_id) const + { + const auto hash = member_id.Hash(); + auto it = std::lower_bound( + members_role.begin(), + members_role.end(), + std::make_pair(hash, std::string())); + + if (it != members_role.end() && (*it).first == hash) + return (*it).second.c_str(); + + return nullptr; + } + + OsmIDTyped id; + AttributesList attributes; + MembersRolesList members_role; }; // It contains data of all parsed relations for each node/way element class ExtractionRelationContainer { public: - using AttributesMap = ExtractionRelation::AttributesMap; + using AttributesMap = ExtractionRelation::AttributesList; using OsmIDTyped = ExtractionRelation::OsmIDTyped; using RelationList = std::vector; + using RelationIDList = std::vector; + using RelationRefMap = std::unordered_map; - void AddRelation(const ExtractionRelation &rel) + void AddRelation(ExtractionRelation && rel) { - BOOST_ASSERT(!rel.is_restriction); - for (auto it : rel.values) - data[it.first].push_back(it.second); + rel.Prepare(); + + BOOST_ASSERT(relations_data.find(rel.id.GetID()) == relations_data.end()); + relations_data.insert(std::make_pair(rel.id.GetID(), std::move(rel))); } - const RelationList &Get(const OsmIDTyped &id) const + void AddRelationMember(const OsmIDTyped & relation_id, const OsmIDTyped & member_id) { - const auto it = data.find(id); - if (it != data.end()) - return it->second; + switch (member_id.GetType()) + { + case osmium::item_type::node: + node_refs[member_id.GetID()].push_back(relation_id); + break; - static RelationList empty; - return empty; + case osmium::item_type::way: + way_refs[member_id.GetID()].push_back(relation_id); + break; + + case osmium::item_type::relation: + rel_refs[member_id.GetID()].push_back(relation_id); + break; + + default: + break; + }; + } + + void Merge(ExtractionRelationContainer && other) + { + for (auto it : other.relations_data) + { + const auto res = relations_data.insert(std::make_pair(it.first, std::move(it.second))); + BOOST_ASSERT(res.second); + (void)res; // prevent unused warning in release + } + + auto MergeRefMap = [&](RelationRefMap & source, RelationRefMap & target) + { + for (auto it : source) + { + auto & v = target[it.first]; + v.insert(v.end(), it.second.begin(), it.second.end()); + } + }; + + MergeRefMap(other.way_refs, way_refs); + MergeRefMap(other.node_refs, node_refs); + MergeRefMap(other.rel_refs, rel_refs); + } + + std::size_t GetRelationsNum() const + { + return relations_data.size(); + } + + const RelationIDList & GetRelations(const OsmIDTyped & member_id) const + { + auto getFromMap = [this](std::uint64_t id, const RelationRefMap & map) -> const RelationIDList & + { + auto it = map.find(id); + if (it != map.end()) + return it->second; + + return empty_rel_list; + }; + + switch (member_id.GetType()) + { + case osmium::item_type::node: + return getFromMap(member_id.GetID(), node_refs); + + case osmium::item_type::way: + return getFromMap(member_id.GetID(), way_refs); + + case osmium::item_type::relation: + return getFromMap(member_id.GetID(), rel_refs); + + default: + break; + } + + return empty_rel_list; + } + + const ExtractionRelation & GetRelationData(const ExtractionRelation::OsmIDTyped & rel_id) const + { + auto it = relations_data.find(rel_id.GetID()); + if (it == relations_data.end()) + throw osrm::util::exception("Can't find relation data for " + std::to_string(rel_id.GetID())); + + return it->second; } private: - std::unordered_map data; + RelationIDList empty_rel_list; + std::unordered_map relations_data; + + // each map contains list of relation id's, that has keyed id as a member + RelationRefMap way_refs; + RelationRefMap node_refs; + RelationRefMap rel_refs; }; } // namespace extractor diff --git a/include/extractor/scripting_environment.hpp b/include/extractor/scripting_environment.hpp index ade5e442a..2a7de2aec 100644 --- a/include/extractor/scripting_environment.hpp +++ b/include/extractor/scripting_environment.hpp @@ -37,7 +37,6 @@ class RestrictionParser; class ExtractionRelationContainer; struct ExtractionNode; struct ExtractionWay; -struct ExtractionRelation; struct ExtractionTurn; struct ExtractionSegment; @@ -59,6 +58,7 @@ class ScriptingEnvironment virtual std::vector GetClassNames() = 0; virtual std::vector GetNameSuffixList() = 0; virtual std::vector GetRestrictions() = 0; + virtual std::vector GetRelations() = 0; virtual void ProcessTurn(ExtractionTurn &turn) = 0; virtual void ProcessSegment(ExtractionSegment &segment) = 0; @@ -68,7 +68,6 @@ class ScriptingEnvironment const ExtractionRelationContainer &relations, std::vector> &resulting_nodes, std::vector> &resulting_ways, - std::vector> &resulting_relations, std::vector &resulting_restrictions) = 0; virtual bool HasLocationDependentData() const = 0; diff --git a/include/extractor/scripting_environment_lua.hpp b/include/extractor/scripting_environment_lua.hpp index 8a00f2048..5cda5c48a 100644 --- a/include/extractor/scripting_environment_lua.hpp +++ b/include/extractor/scripting_environment_lua.hpp @@ -29,11 +29,10 @@ struct LuaScriptingContext final void ProcessNode(const osmium::Node &, ExtractionNode &result, - const ExtractionRelationContainer::RelationList &relations); + const ExtractionRelationContainer &relations); void ProcessWay(const osmium::Way &, ExtractionWay &result, - const ExtractionRelationContainer::RelationList &relations); - void ProcessRelation(const osmium::Relation &, ExtractionRelation &result); + const ExtractionRelationContainer &relations); ProfileProperties properties; RasterContainer raster_sources; @@ -42,13 +41,11 @@ struct LuaScriptingContext final bool has_turn_penalty_function; bool has_node_function; bool has_way_function; - bool has_relation_function; bool has_segment_function; sol::function turn_function; sol::function way_function; sol::function node_function; - sol::function relation_function; sol::function segment_function; int api_version; @@ -84,6 +81,7 @@ class Sol2ScriptingEnvironment final : public ScriptingEnvironment std::vector GetNameSuffixList() override; std::vector GetClassNames() override; std::vector GetRestrictions() override; + std::vector GetRelations() override; void ProcessTurn(ExtractionTurn &turn) override; void ProcessSegment(ExtractionSegment &segment) override; @@ -93,7 +91,6 @@ class Sol2ScriptingEnvironment final : public ScriptingEnvironment const ExtractionRelationContainer &relations, std::vector> &resulting_nodes, std::vector> &resulting_ways, - std::vector> &resulting_relations, std::vector &resulting_restrictions) override; bool HasLocationDependentData() const override { return !location_dependent_data.empty(); } diff --git a/profiles/car.lua b/profiles/car.lua index 72b41acb5..1cad4e35c 100644 --- a/profiles/car.lua +++ b/profiles/car.lua @@ -281,6 +281,10 @@ function setup() ['za:urban'] = 60, ['za:rural'] = 100, ["none"] = 140 + }, + + relation_types = Sequence { + "route" } } end diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index b12750d9b..78fc981e2 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -386,6 +386,10 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, turn_lane_map, scripting_environment.GetProfileProperties()); + // get list of supported relation types + auto relation_types = scripting_environment.GetRelations(); + std::sort(relation_types.begin(), relation_types.end()); + ExtractionRelationContainer relations; std::vector restrictions = scripting_environment.GetRestrictions(); // setup restriction parser @@ -445,7 +449,6 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, relations, parsed_buffer.resulting_nodes, parsed_buffer.resulting_ways, - parsed_buffer.resulting_relations, parsed_buffer.resulting_restrictions); return parsed_buffer; }); @@ -453,6 +456,7 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, // Parsed nodes and ways handler unsigned number_of_nodes = 0; unsigned number_of_ways = 0; + unsigned number_of_restrictions = 0; tbb::filter_t buffer_storage( tbb::filter::serial_in_order, [&](const ParsedBuffer &parsed_buffer) { @@ -467,29 +471,57 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, { extractor_callbacks->ProcessWay(result.first, result.second); } - }); - - // Parsed relations handler - unsigned number_of_relations = 0; - tbb::filter_t buffer_storage_relation( - tbb::filter::serial_in_order, [&](const ParsedBuffer &parsed_buffer) { - - number_of_relations += parsed_buffer.resulting_relations.size(); - for (const auto &result : parsed_buffer.resulting_relations) - { - /// TODO: add restriction processing - if (result.second.is_restriction) - continue; - - relations.AddRelation(result.second); - } + number_of_restrictions += parsed_buffer.resulting_restrictions.size(); for (const auto &result : parsed_buffer.resulting_restrictions) { extractor_callbacks->ProcessRestriction(result); } }); + + + tbb::filter_t> buffer_relation_cache( + tbb::filter::parallel, [&](const SharedBuffer buffer) { + if (!buffer) + return std::shared_ptr{}; + + auto relations = std::make_shared(); + for (auto entity = buffer->cbegin(), end = buffer->cend(); entity != end; ++entity) + { + if (entity->type() != osmium::item_type::relation) + continue; + + const auto & rel = static_cast(*entity); + + const char * rel_type = rel.get_value_by_key("type"); + if (!rel_type || !std::binary_search(relation_types.begin(), relation_types.end(), std::string(rel_type))) + continue; + + ExtractionRelation extracted_rel({rel.id(), osmium::item_type::relation}); + for (auto const & t : rel.tags()) + extracted_rel.attributes.emplace_back(std::make_pair(t.key(), t.value())); + + for (auto const & m : rel.members()) + { + ExtractionRelation::OsmIDTyped const mid(m.ref(), m.type()); + extracted_rel.AddMember(mid, m.role()); + relations->AddRelationMember(extracted_rel.id, mid); + } + + relations->AddRelation(std::move(extracted_rel)); + }; + return relations; + }); + + unsigned number_of_relations = 0; + tbb::filter_t, void> buffer_storage_relation( + tbb::filter::serial_in_order, [&](const std::shared_ptr parsed_relations) { + + number_of_relations += parsed_relations->GetRelationsNum(); + relations.Merge(std::move(*parsed_relations)); + }); + // Parse OSM elements with parallel transformer // Number of pipeline tokens that yielded the best speedup was about 1.5 * num_cores const auto num_threads = tbb::task_scheduler_init::default_num_threads() * 1.5; @@ -500,13 +532,13 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, util::Log() << "Parse relations ..."; osmium::io::Reader reader(input_file, osmium::osm_entity_bits::relation, read_meta); tbb::parallel_pipeline( - num_threads, buffer_reader(reader) & buffer_transformer & buffer_storage_relation); + num_threads, buffer_reader(reader) & buffer_relation_cache & buffer_storage_relation); } { // Nodes and ways reading pipeline util::Log() << "Parse ways and nodes ..."; osmium::io::Reader reader( - input_file, osmium::osm_entity_bits::node | osmium::osm_entity_bits::way, read_meta); + input_file, osmium::osm_entity_bits::node | osmium::osm_entity_bits::way | osmium::osm_entity_bits::relation, read_meta); const auto pipeline = scripting_environment.HasLocationDependentData() && config.use_locations_cache @@ -519,7 +551,8 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, util::Log() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds"; util::Log() << "Raw input contains " << number_of_nodes << " nodes, " << number_of_ways - << " ways, and " << number_of_relations << " relations"; + << " ways, and " << number_of_relations << " relations, " << number_of_restrictions + << " restrictions"; extractor_callbacks.reset(); diff --git a/src/extractor/scripting_environment_lua.cpp b/src/extractor/scripting_environment_lua.cpp index 1b9735192..0117b380c 100644 --- a/src/extractor/scripting_environment_lua.cpp +++ b/src/extractor/scripting_environment_lua.cpp @@ -323,29 +323,6 @@ void Sol2ScriptingEnvironment::InitContext(LuaScriptingContext &context) return boost::apply_visitor(to_lua_object(context.state), value); }); - context.state.new_usertype( - "RelationMember", - "role", - &osmium::RelationMember::role, - "type", - &osmium::RelationMember::type, - "id", - [](const osmium::RelationMember &member) -> osmium::object_id_type { - return member.ref(); - }); - - context.state.new_usertype( - "Relation", - "get_value_by_key", - &get_value_by_key, - "id", - &osmium::Relation::id, - "version", - &osmium::Relation::version, - "members", - [](const osmium::Relation &rel) -> const osmium::RelationMemberList & { - return rel.members(); - }); context.state.new_usertype("Node", "location", @@ -440,17 +417,59 @@ void Sol2ScriptingEnvironment::InitContext(LuaScriptingContext &context) sol::property([](const ExtractionWay &way) { return way.is_left_hand_driving; }, [](ExtractionWay &way, bool flag) { way.is_left_hand_driving = flag; })); + auto getTypedRefBySol = [](const sol::object & obj) -> ExtractionRelation::OsmIDTyped + { + if (obj.is()) + { + osmium::Way * way = obj.as(); + return { way->id(), osmium::item_type::way }; + } + + if (obj.is()) + { + osmium::Relation * rel = obj.as(); + return { rel->id(), osmium::item_type::relation }; + } + + if (obj.is()) + { + osmium::Node * node = obj.as(); + return { node->id(), osmium::item_type::node }; + } + + return ExtractionRelation::OsmIDTyped(0, osmium::item_type::undefined); + }; + + context.state.new_usertype( + "OsmIDTyped", + "id", + &ExtractionRelation::OsmIDTyped::GetID, + "type", + &ExtractionRelation::OsmIDTyped::GetType); + context.state.new_usertype( "ExtractionRelation", - sol::meta_function::new_index, - [](ExtractionRelation &rel, const osmium::RelationMember &member) - -> ExtractionRelation::AttributesMap & { return rel.GetMember(member); }, - sol::meta_function::index, - [](ExtractionRelation &rel, const osmium::RelationMember &member) - -> ExtractionRelation::AttributesMap & { return rel.GetMember(member); }, - "restriction", - sol::property([](const ExtractionRelation &rel) { return rel.is_restriction; }, - [](ExtractionRelation &rel, bool flag) { rel.is_restriction = flag; })); + "get_value_by_key", + [](ExtractionRelation &rel, const char * key) -> const char * { return rel.GetAttr(key); }, + "get_role", + [&getTypedRefBySol](ExtractionRelation &rel, const sol::object & obj) -> const char * + { + return rel.GetRole(getTypedRefBySol(obj)); + }); + + context.state.new_usertype( + "ExtractionRelationContainer", + "get_relations", + [&getTypedRefBySol](ExtractionRelationContainer &cont, const sol::object & obj) + -> const ExtractionRelationContainer::RelationIDList & + { + return cont.GetRelations(getTypedRefBySol(obj)); + }, + "relation", + [](ExtractionRelationContainer &cont, const ExtractionRelation::OsmIDTyped & rel_id) -> const ExtractionRelation & + { + return cont.GetRelationData(rel_id); + }); context.state.new_usertype("ExtractionSegment", "source", @@ -633,14 +652,6 @@ void Sol2ScriptingEnvironment::InitContext(LuaScriptingContext &context) switch (context.api_version) { case 3: - { - initV2Context(); - context.relation_function = function_table.value()["process_relation"]; - - context.has_relation_function = context.relation_function.valid(); - break; - } - case 2: { initV2Context(); @@ -719,12 +730,10 @@ void Sol2ScriptingEnvironment::ProcessElements( const ExtractionRelationContainer &relations, std::vector> &resulting_nodes, std::vector> &resulting_ways, - std::vector> &resulting_relations, std::vector &resulting_restrictions) { ExtractionNode result_node; ExtractionWay result_way; - ExtractionRelation result_relation; auto &local_context = this->GetSol2Context(); for (auto entity = buffer.cbegin(), end = buffer.cend(); entity != end; ++entity) @@ -738,8 +747,7 @@ void Sol2ScriptingEnvironment::ProcessElements( if (local_context.has_node_function && (!node.tags().empty() || local_context.properties.call_tagless_node_function)) { - const auto &id = ExtractionRelation::OsmIDTyped(node.id(), osmium::item_type::node); - local_context.ProcessNode(node, result_node, relations.Get(id)); + local_context.ProcessNode(node, result_node, relations); } resulting_nodes.push_back({node, std::move(result_node)}); } @@ -750,8 +758,7 @@ void Sol2ScriptingEnvironment::ProcessElements( result_way.clear(); if (local_context.has_way_function) { - const auto &id = ExtractionRelation::OsmIDTyped(way.id(), osmium::item_type::way); - local_context.ProcessWay(way, result_way, relations.Get(id)); + local_context.ProcessWay(way, result_way, relations); } resulting_ways.push_back({way, std::move(result_way)}); } @@ -763,16 +770,6 @@ void Sol2ScriptingEnvironment::ProcessElements( { resulting_restrictions.push_back(*result_res); } - - if (local_context.api_version > 2) - { - result_relation.clear(); - if (local_context.has_relation_function) - { - local_context.ProcessRelation(relation, result_relation); - } - resulting_relations.push_back({relation, std::move(result_relation)}); - } } break; default: @@ -901,6 +898,18 @@ std::vector Sol2ScriptingEnvironment::GetRestrictions() } } +std::vector Sol2ScriptingEnvironment::GetRelations() +{ + auto &context = GetSol2Context(); + switch (context.api_version) + { + case 3: + return Sol2ScriptingEnvironment::GetStringListFromTable("relation_types"); + default: + return {}; + } +} + void Sol2ScriptingEnvironment::ProcessTurn(ExtractionTurn &turn) { auto &context = GetSol2Context(); @@ -992,7 +1001,7 @@ void Sol2ScriptingEnvironment::ProcessSegment(ExtractionSegment &segment) void LuaScriptingContext::ProcessNode(const osmium::Node &node, ExtractionNode &result, - const ExtractionRelationContainer::RelationList &relations) + const ExtractionRelationContainer &relations) { BOOST_ASSERT(state.lua_state() != nullptr); @@ -1013,7 +1022,7 @@ void LuaScriptingContext::ProcessNode(const osmium::Node &node, void LuaScriptingContext::ProcessWay(const osmium::Way &way, ExtractionWay &result, - const ExtractionRelationContainer::RelationList &relations) + const ExtractionRelationContainer &relations) { BOOST_ASSERT(state.lua_state() != nullptr); @@ -1033,14 +1042,5 @@ void LuaScriptingContext::ProcessWay(const osmium::Way &way, } } -void LuaScriptingContext::ProcessRelation(const osmium::Relation &relation, - ExtractionRelation &result) -{ - BOOST_ASSERT(state.lua_state() != nullptr); - BOOST_ASSERT(api_version > 2); - - relation_function(profile_table, relation, result); -} - } // namespace extractor } // namespace osrm