diff --git a/Extractor/Extractor.cpp b/Extractor/Extractor.cpp index c5e3817cf..13e82ba29 100644 --- a/Extractor/Extractor.cpp +++ b/Extractor/Extractor.cpp @@ -49,12 +49,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include +#include #include #include #include +#include #include #include #include @@ -116,16 +118,22 @@ int Extractor::Run(int argc, char *argv[]) SimpleLogger().Write() << "Input file: " << extractor_config.input_path.filename().string(); SimpleLogger().Write() << "Profile: " << extractor_config.profile_path.filename().string(); SimpleLogger().Write() << "Threads: " << extractor_config.requested_num_threads; - if (recommended_num_threads != extractor_config.requested_num_threads) - { - SimpleLogger().Write(logWARNING) << "The recommended number of threads is " - << recommended_num_threads - << "! This setting may have performance side-effects."; - } + // if (recommended_num_threads != extractor_config.requested_num_threads) + // { + // SimpleLogger().Write(logWARNING) << "The recommended number of threads is " + // << recommended_num_threads + // << "! This setting may have performance side-effects."; + // } - tbb::task_scheduler_init init(extractor_config.requested_num_threads); + auto number_of_threads = std::max(1, + std::min(static_cast(recommended_num_threads), static_cast(extractor_config.requested_num_threads)) ); - /*** Setup Scripting Environment ***/ + tbb::task_scheduler_init init(number_of_threads); + + SimpleLogger().Write() << "requested_num_threads: " << extractor_config.requested_num_threads; + SimpleLogger().Write() << "number_of_threads: " << number_of_threads; + + // setup scripting environment ScriptingEnvironment scripting_environment(extractor_config.profile_path.string().c_str()); std::unordered_map string_map; @@ -166,48 +174,94 @@ int Extractor::Run(int argc, char *argv[]) timestamp_out.write(timestamp.c_str(), timestamp.length()); timestamp_out.close(); - lua_State *lua_state = scripting_environment.getLuaState(); + // lua_State *lua_state = scripting_environment.getLuaState(); luabind::set_pcall_callback(&lua_error_callback); - RestrictionParser restriction_parser(scripting_environment); - - ExtractionNode result_node; - ExtractionWay result_way; + // initialize vectors holding parsed objects + tbb::concurrent_vector> resulting_nodes; + tbb::concurrent_vector> resulting_ways; + tbb::concurrent_vector> resulting_restrictions; while (osmium::memory::Buffer buffer = reader.read()) { - for (osmium::OSMEntity &entity : buffer) + // create a vector of iterators into the buffer + std::vector elements; + osmium::memory::Buffer::iterator iter = std::begin(buffer); + while(iter != std::end(buffer)) { - switch (entity.type()) + elements.push_back(iter); + iter = std::next(iter); + } + + // clear resulting vectors + resulting_nodes.clear(); + resulting_ways.clear(); + resulting_restrictions.clear(); + + // SimpleLogger().Write(logDEBUG) << "elements count: " << elements.size(); + + // parse OSM entities in parallel, store in resulting vectors + tbb::parallel_for(tbb::blocked_range(0, elements.size()), + [&](const tbb::blocked_range& range) + { + for (auto x = range.begin(); x != range.end(); ++x) + { + auto entity = elements[x]; + + ExtractionNode result_node; + ExtractionWay result_way; + // RestrictionParser restriction_parser(scripting_environment); + + switch (entity->type()) { case osmium::item_type::node: ++number_of_nodes; result_node.Clear(); - luabind::call_function(lua_state, + luabind::call_function(scripting_environment.getLuaState(), "node_function", - boost::cref(static_cast(entity)), + boost::cref(static_cast(*entity)), boost::ref(result_node)); - extractor_callbacks->ProcessNode(static_cast(entity), - result_node); + resulting_nodes.emplace_back(x, result_node); + // extractor_callbacks->ProcessNode(static_cast(*entity), + // result_node); break; case osmium::item_type::way: ++number_of_ways; result_way.Clear(); - luabind::call_function(lua_state, + luabind::call_function(scripting_environment.getLuaState(), "way_function", - boost::cref(static_cast(entity)), + boost::cref(static_cast(*entity)), boost::ref(result_way)); - extractor_callbacks->ProcessWay(static_cast(entity), result_way); + resulting_ways.emplace_back(x, result_way); + // extractor_callbacks->ProcessWay(static_cast(*entity), result_way); break; case osmium::item_type::relation: ++number_of_relations; - extractor_callbacks->ProcessRestriction( - restriction_parser.TryParse(static_cast(entity))); + // resulting_restrictions.emplace_back(restriction_parser.TryParse(static_cast(*entity))); + // extractor_callbacks->ProcessRestriction(restriction_parser.TryParse(static_cast(*entity))); break; default: ++number_of_others; break; } + } + } + ); + + // put parsed objects thru extractor callbacks + for (const auto &result : resulting_nodes) + { + extractor_callbacks->ProcessNode(static_cast(*(elements[result.first])), + result.second); + } + for (const auto &result : resulting_ways) + { + extractor_callbacks->ProcessWay(static_cast(*(elements[result.first])), + result.second); + } + for (const auto &result : resulting_restrictions) + { + extractor_callbacks->ProcessRestriction(result); } } TIMER_STOP(parsing); diff --git a/Extractor/ExtractorCallbacks.cpp b/Extractor/ExtractorCallbacks.cpp index df56d7fd3..5f398dc8a 100644 --- a/Extractor/ExtractorCallbacks.cpp +++ b/Extractor/ExtractorCallbacks.cpp @@ -66,10 +66,10 @@ void ExtractorCallbacks::ProcessRestriction( if (restriction) { external_memory.restrictions_list.push_back(restriction.get()); - SimpleLogger().Write() << "from: " << restriction.get().restriction.from.node << - ",via: " << restriction.get().restriction.via.node << - ", to: " << restriction.get().restriction.to.node << - ", only: " << (restriction.get().restriction.flags.is_only ? "y" : "n"); + // SimpleLogger().Write() << "from: " << restriction.get().restriction.from.node << + // ",via: " << restriction.get().restriction.via.node << + // ", to: " << restriction.get().restriction.to.node << + // ", only: " << (restriction.get().restriction.flags.is_only ? "y" : "n"); } } /** warning: caller needs to take care of synchronization! */ diff --git a/Extractor/RestrictionParser.cpp b/Extractor/RestrictionParser.cpp index 690ad82ac..ac0931a24 100644 --- a/Extractor/RestrictionParser.cpp +++ b/Extractor/RestrictionParser.cpp @@ -51,18 +51,18 @@ int lua_error_callback(lua_State *L) } } -RestrictionParser::RestrictionParser(ScriptingEnvironment &scripting_environment) - : lua_state(scripting_environment.getLuaState()), use_turn_restrictions(true) +RestrictionParser::RestrictionParser(lua_State *lua_state) + : /*lua_state(scripting_environment.getLuaState()),*/ use_turn_restrictions(true) { - ReadUseRestrictionsSetting(); + ReadUseRestrictionsSetting(lua_state); if (use_turn_restrictions) { - ReadRestrictionExceptions(); + ReadRestrictionExceptions(lua_state); } } -void RestrictionParser::ReadUseRestrictionsSetting() +void RestrictionParser::ReadUseRestrictionsSetting(lua_State *lua_state) { if (0 == luaL_dostring(lua_state, "return use_turn_restrictions\n")) { @@ -82,7 +82,7 @@ void RestrictionParser::ReadUseRestrictionsSetting() } } -void RestrictionParser::ReadRestrictionExceptions() +void RestrictionParser::ReadRestrictionExceptions(lua_State *lua_state) { if (lua_function_exists(lua_state, "get_exceptions")) { @@ -104,7 +104,7 @@ void RestrictionParser::ReadRestrictionExceptions() } } -mapbox::util::optional RestrictionParser::TryParse(osmium::Relation &relation) const +mapbox::util::optional RestrictionParser::TryParse(lua_State *lua_state, osmium::Relation &relation) const { // return if turn restrictions should be ignored if (!use_turn_restrictions) diff --git a/Extractor/RestrictionParser.h b/Extractor/RestrictionParser.h index e428e6166..fcf98dfc5 100644 --- a/Extractor/RestrictionParser.h +++ b/Extractor/RestrictionParser.h @@ -44,16 +44,16 @@ class ScriptingEnvironment; class RestrictionParser { public: - RestrictionParser(ScriptingEnvironment &scripting_environment); + // RestrictionParser(ScriptingEnvironment &scripting_environment); + RestrictionParser(lua_State *lua_state); + mapbox::util::optional TryParse(lua_State *lua_state, osmium::Relation& relation) const; - mapbox::util::optional TryParse(osmium::Relation& relation) const; - - void ReadUseRestrictionsSetting(); - void ReadRestrictionExceptions(); private: - bool ShouldIgnoreRestriction(const std::string &except_tag_string) const; + void ReadUseRestrictionsSetting(lua_State *lua_state); + void ReadRestrictionExceptions(lua_State *lua_state); + bool ShouldIgnoreRestriction(lua_State *lua_state, const std::string &except_tag_string) const; - lua_State *lua_state; + // lua_State *lua_state; std::vector restriction_exceptions; bool use_turn_restrictions; };