From f79bcc6b8dea67ab9b8305c69dbaf78ae06faca2 Mon Sep 17 00:00:00 2001 From: Denis Koronchik Date: Wed, 30 Aug 2017 19:35:38 +0300 Subject: [PATCH] Pass relation data to way and node functions --- include/extractor/extraction_relation.hpp | 35 +++++++++++-- include/extractor/extractor_callbacks.hpp | 3 ++ include/extractor/scripting_environment.hpp | 2 + .../extractor/scripting_environment_lua.hpp | 6 ++- src/extractor/extractor.cpp | 49 +++++++++++++++---- src/extractor/scripting_environment_lua.cpp | 21 +++++--- .../mocks/mock_scripting_environment.hpp | 3 +- 7 files changed, 94 insertions(+), 25 deletions(-) diff --git a/include/extractor/extraction_relation.hpp b/include/extractor/extraction_relation.hpp index f4ec39d90..98d2f515e 100644 --- a/include/extractor/extraction_relation.hpp +++ b/include/extractor/extraction_relation.hpp @@ -5,6 +5,7 @@ #include #include +#include namespace osrm { @@ -45,15 +46,39 @@ struct ExtractionRelation return values[id.Hash()]; } -// AttributesMap & operator[] (util::OsmIDTyped id) -// { -// return values[id]; -// } - bool is_restriction; std::unordered_map values; }; +// It contains data of all parsed relations for each node/way element +class ExtractionRelationContainer +{ +public: + using AttributesMap = ExtractionRelation::AttributesMap; + using RelationList = std::vector; + + void AddRelation(const ExtractionRelation & rel) + { + BOOST_ASSERT(!rel.is_restriction); + for (auto it : rel.values) + data[it.first].push_back(it.second); + } + + const RelationList & Get(const util::OsmIDTyped & id) const + { + const auto it = data.find(id.Hash()); + if (it != data.end()) + return it->second; + + static RelationList empty; + return empty; + } + +private: + // TODO: need to store more common data + std::unordered_map data; +}; + } // namespace extractor } // namespace osrm diff --git a/include/extractor/extractor_callbacks.hpp b/include/extractor/extractor_callbacks.hpp index 4bdc05d51..e91fd3d04 100644 --- a/include/extractor/extractor_callbacks.hpp +++ b/include/extractor/extractor_callbacks.hpp @@ -15,6 +15,7 @@ namespace osmium { class Node; class Way; +class Relation; } namespace std @@ -44,6 +45,7 @@ namespace extractor class ExtractionContainers; struct ExtractionNode; struct ExtractionWay; +struct ExtractionRelation; struct ProfileProperties; struct InputConditionalTurnRestriction; @@ -88,6 +90,7 @@ class ExtractorCallbacks // warning: caller needs to take care of synchronization! void ProcessWay(const osmium::Way ¤t_way, const ExtractionWay &result_way); }; + } } diff --git a/include/extractor/scripting_environment.hpp b/include/extractor/scripting_environment.hpp index 8a686e39d..b93672365 100644 --- a/include/extractor/scripting_environment.hpp +++ b/include/extractor/scripting_environment.hpp @@ -34,6 +34,7 @@ namespace extractor { class RestrictionParser; +class ExtractionRelationContainer; struct ExtractionNode; struct ExtractionWay; struct ExtractionRelation; @@ -64,6 +65,7 @@ class ScriptingEnvironment virtual void ProcessElements(const osmium::memory::Buffer &buffer, const RestrictionParser &restriction_parser, + const ExtractionRelationContainer &relations, std::vector> &resulting_nodes, std::vector> &resulting_ways, std::vector> &resulting_relations, diff --git a/include/extractor/scripting_environment_lua.hpp b/include/extractor/scripting_environment_lua.hpp index 95b7e3b23..444244305 100644 --- a/include/extractor/scripting_environment_lua.hpp +++ b/include/extractor/scripting_environment_lua.hpp @@ -3,6 +3,7 @@ #include "extractor/raster_source.hpp" #include "extractor/scripting_environment.hpp" +#include "extractor/extraction_relation.hpp" #include @@ -19,8 +20,8 @@ namespace extractor struct LuaScriptingContext final { - void ProcessNode(const osmium::Node &, ExtractionNode &result); - void ProcessWay(const osmium::Way &, ExtractionWay &result); + void ProcessNode(const osmium::Node &, ExtractionNode &result, const ExtractionRelationContainer::RelationList &relations); + void ProcessWay(const osmium::Way &, ExtractionWay &result, const ExtractionRelationContainer::RelationList &relations); void ProcessRelation(const osmium::Relation &, ExtractionRelation &result); ProfileProperties properties; @@ -71,6 +72,7 @@ class Sol2ScriptingEnvironment final : public ScriptingEnvironment void ProcessElements(const osmium::memory::Buffer &buffer, const RestrictionParser &restriction_parser, + const ExtractionRelationContainer &relations, std::vector> &resulting_nodes, std::vector> &resulting_ways, std::vector> &resulting_relations, diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index 61e16662c..d331bfa7e 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -289,12 +289,12 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, const osmium::io::File input_file(config.input_path.string()); osmium::thread::Pool pool(number_of_threads); - osmium::io::Reader reader( - input_file, - pool, - (config.use_metadata ? osmium::io::read_meta::yes : osmium::io::read_meta::no)); - const osmium::io::Header header = reader.header(); + std::unique_ptr reader( + new osmium::io::Reader(input_file, osmium::osm_entity_bits::relation, + (config.use_metadata ? osmium::io::read_meta::yes : osmium::io::read_meta::no))); + + osmium::io::Header header = reader->header(); unsigned number_of_nodes = 0; unsigned number_of_ways = 0; @@ -332,6 +332,7 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, timestamp_file.WriteFrom(timestamp.c_str(), timestamp.length()); + ExtractionRelationContainer relations; std::vector restrictions = scripting_environment.GetRestrictions(); // setup restriction parser const RestrictionParser restriction_parser( @@ -339,8 +340,6 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, config.parse_conditionals, restrictions); - std::mutex process_mutex; - using SharedBuffer = std::shared_ptr; struct ParsedBuffer { @@ -353,7 +352,7 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, tbb::filter_t buffer_reader( tbb::filter::serial_in_order, [&](tbb::flow_control &fc) { - if (auto buffer = reader.read()) + if (auto buffer = reader->read()) { return std::make_shared(std::move(buffer)); } @@ -372,6 +371,7 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, parsed_buffer->buffer = buffer; scripting_environment.ProcessElements(*buffer, restriction_parser, + relations, parsed_buffer->resulting_nodes, parsed_buffer->resulting_ways, parsed_buffer->resulting_relations, @@ -394,13 +394,44 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, { extractor_callbacks->ProcessWay(result.first, result.second); } - number_of_relations += parsed_buffer->resulting_restrictions.size(); + }); + + tbb::filter_t, void> buffer_storage_relation( + tbb::filter::serial_in_order, [&](const std::shared_ptr parsed_buffer) { + if (!parsed_buffer) + return; + + number_of_relations += parsed_buffer->resulting_relations.size(); + for (const auto &result : parsed_buffer->resulting_relations) + { + /// TODO: add restriction processing + if (result.second.is_restriction) + continue; + + relations.AddRelation(result.second); + } + for (const auto &result : parsed_buffer->resulting_restrictions) { extractor_callbacks->ProcessRestriction(result); } }); + /* Main trick that we can use the same pipeline. It just receive relation objects + * from osmium. So other containers would be empty and doesn't process anything + */ + util::Log() << "Parse relations ..."; + tbb::parallel_pipeline(tbb::task_scheduler_init::default_num_threads() * 1.5, + buffer_reader & buffer_transform & buffer_storage_relation); + reader->close(); + + /* At this step we just filter ways and nodes from osmium, so any relation wouldn't be + * processed there. + */ + util::Log() << "Parse ways and nodes ..."; + reader.reset(new osmium::io::Reader(input_file, osmium::osm_entity_bits::node | osmium::osm_entity_bits::way, + (config.use_metadata ? osmium::io::read_meta::yes : osmium::io::read_meta::no))); + // Number of pipeline tokens that yielded the best speedup was about 1.5 * num_cores tbb::parallel_pipeline(tbb::task_scheduler_init::default_num_threads() * 1.5, buffer_reader & buffer_transform & buffer_storage); diff --git a/src/extractor/scripting_environment_lua.cpp b/src/extractor/scripting_environment_lua.cpp index 28b04be68..8381be937 100644 --- a/src/extractor/scripting_environment_lua.cpp +++ b/src/extractor/scripting_environment_lua.cpp @@ -737,6 +737,7 @@ LuaScriptingContext &Sol2ScriptingEnvironment::GetSol2Context() void Sol2ScriptingEnvironment::ProcessElements( const osmium::memory::Buffer &buffer, const RestrictionParser &restriction_parser, + const ExtractionRelationContainer &relations, std::vector> &resulting_nodes, std::vector> &resulting_ways, std::vector> &resulting_relations, @@ -757,7 +758,9 @@ void Sol2ScriptingEnvironment::ProcessElements( (!static_cast(*entity).tags().empty() || local_context.properties.call_tagless_node_function)) { - local_context.ProcessNode(static_cast(*entity), result_node); + const osmium::Node & node = static_cast(*entity); + const util::OsmIDTyped id(node.id(), std::uint8_t(node.type())); + local_context.ProcessNode(node, result_node, relations.Get(id)); } resulting_nodes.push_back(std::pair( static_cast(*entity), std::move(result_node))); @@ -766,7 +769,9 @@ void Sol2ScriptingEnvironment::ProcessElements( result_way.clear(); if (local_context.has_way_function) { - local_context.ProcessWay(static_cast(*entity), result_way); + const osmium::Way & way = static_cast(*entity); + const util::OsmIDTyped id(way.id(), std::uint8_t(way.type())); + local_context.ProcessWay(way, result_way, relations.Get(id)); } resulting_ways.push_back(std::pair( static_cast(*entity), std::move(result_way))); @@ -1007,15 +1012,15 @@ void Sol2ScriptingEnvironment::ProcessSegment(ExtractionSegment &segment) } } -void LuaScriptingContext::ProcessNode(const osmium::Node &node, ExtractionNode &result) +void LuaScriptingContext::ProcessNode(const osmium::Node &node, ExtractionNode &result, const ExtractionRelationContainer::RelationList &relations) { BOOST_ASSERT(state.lua_state() != nullptr); switch (api_version) { case 3: -// BOOST_ASSERT(false); // TODO: implement me -// break; + node_function(profile_table, node, result, relations); + break; case 2: node_function(profile_table, node, result); break; @@ -1026,15 +1031,15 @@ void LuaScriptingContext::ProcessNode(const osmium::Node &node, ExtractionNode & } } -void LuaScriptingContext::ProcessWay(const osmium::Way &way, ExtractionWay &result) +void LuaScriptingContext::ProcessWay(const osmium::Way &way, ExtractionWay &result, const ExtractionRelationContainer::RelationList &relations) { BOOST_ASSERT(state.lua_state() != nullptr); switch (api_version) { case 3: -// BOOST_ASSERT(false); // TODO: implement me -// break; + way_function(profile_table, way, result, relations); + break; case 2: way_function(profile_table, way, result); break; diff --git a/unit_tests/mocks/mock_scripting_environment.hpp b/unit_tests/mocks/mock_scripting_environment.hpp index 3f53e1521..088379cfa 100644 --- a/unit_tests/mocks/mock_scripting_environment.hpp +++ b/unit_tests/mocks/mock_scripting_environment.hpp @@ -36,9 +36,10 @@ class MockScriptingEnvironment : public extractor::ScriptingEnvironment void ProcessElements(const osmium::memory::Buffer &, const extractor::RestrictionParser &, + const extractor::ExtractionRelationContainer &, std::vector> &, std::vector> &, - std::vector> &resulting_relations, + std::vector> &, std::vector &) override final { }