diff --git a/Extractor/PBFParser.cpp b/Extractor/PBFParser.cpp index b3331348f..4ad8ba617 100644 --- a/Extractor/PBFParser.cpp +++ b/Extractor/PBFParser.cpp @@ -40,6 +40,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../typedefs.h" #include +#include #include @@ -51,9 +52,19 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PBFParser::PBFParser(const char *fileName, ExtractorCallbacks *extractor_callbacks, - ScriptingEnvironment &scripting_environment) + ScriptingEnvironment &scripting_environment, + unsigned num_threads) : BaseParser(extractor_callbacks, scripting_environment) { + if (0 == num_threads) + { + num_parser_threads = tbb::task_scheduler_init::default_num_threads(); + } + else + { + num_parser_threads = num_threads; + } + GOOGLE_PROTOBUF_VERIFY_VERSION; // TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk? // NOTE: With Lua scripting, it is parsing the stuff. I/O is virtually for free. @@ -160,6 +171,8 @@ inline void PBFParser::ReadData() inline void PBFParser::ParseData() { + tbb::task_scheduler_init init(num_parser_threads); + while (true) { ParserThreadData *thread_data; diff --git a/Extractor/PBFParser.h b/Extractor/PBFParser.h index 1342e914e..25f41734c 100644 --- a/Extractor/PBFParser.h +++ b/Extractor/PBFParser.h @@ -63,7 +63,10 @@ class PBFParser : public BaseParser }; public: - PBFParser(const char *file_name, ExtractorCallbacks *extractor_callbacks, ScriptingEnvironment &scripting_environment); + PBFParser(const char *file_name, + ExtractorCallbacks *extractor_callbacks, + ScriptingEnvironment &scripting_environment, + unsigned num_parser_threads=0); virtual ~PBFParser(); inline bool ReadHeader(); @@ -94,6 +97,7 @@ class PBFParser : public BaseParser std::fstream input; // the input stream to parse std::shared_ptr> thread_data_queue; + unsigned num_parser_threads; }; #endif /* PBFPARSER_H_ */ diff --git a/extractor.cpp b/extractor.cpp index 6e22b76f7..cb2bd991a 100644 --- a/extractor.cpp +++ b/extractor.cpp @@ -81,7 +81,7 @@ int main(int argc, char *argv[]) &profile_path)->default_value("profile.lua"), "Path to LUA routing profile")( "threads,t", - boost::program_options::value(&requested_num_threads)->default_value(8), + boost::program_options::value(&requested_num_threads)->default_value(tbb::task_scheduler_init::default_num_threads()), "Number of threads to use"); // hidden options, will be allowed both on command line and in config file, but will not be @@ -166,15 +166,19 @@ int main(int argc, char *argv[]) return 1; } - unsigned hardware_threads = std::max(1u, std::thread::hardware_concurrency()); - unsigned real_num_threads = std::min(hardware_threads, requested_num_threads); + const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); SimpleLogger().Write() << "Input file: " << input_path.filename().string(); SimpleLogger().Write() << "Profile: " << profile_path.filename().string(); - SimpleLogger().Write() << "Threads: " << real_num_threads << " (requested " - << requested_num_threads << ")"; + SimpleLogger().Write() << "Threads: " << requested_num_threads; + if (recommended_num_threads != requested_num_threads) + { + SimpleLogger().Write(logWARNING) << "The recommended number of threads is " + << recommended_num_threads + << "! This setting may have performance side-effects."; + } - tbb::task_scheduler_init init(real_num_threads); + tbb::task_scheduler_init init(requested_num_threads); /*** Setup Scripting Environment ***/ ScriptingEnvironment scripting_environment(profile_path.string().c_str()); @@ -229,7 +233,10 @@ int main(int argc, char *argv[]) BaseParser *parser; if (file_has_pbf_format) { - parser = new PBFParser(input_path.string().c_str(), extractor_callbacks, scripting_environment); + parser = new PBFParser(input_path.string().c_str(), + extractor_callbacks, + scripting_environment, + requested_num_threads); } else { diff --git a/prepare.cpp b/prepare.cpp index 98f0f475f..bb0dab455 100644 --- a/prepare.cpp +++ b/prepare.cpp @@ -99,7 +99,7 @@ int main(int argc, char *argv[]) ->default_value("profile.lua"), "Path to LUA routing profile")( "threads,t", - boost::program_options::value(&requested_num_threads)->default_value(8), + boost::program_options::value(&requested_num_threads)->default_value(tbb::task_scheduler_init::default_num_threads()), "Number of threads to use"); // hidden options, will be allowed both on command line and in config file, but will not be @@ -178,16 +178,20 @@ int main(int argc, char *argv[]) return 1; } - unsigned int hardware_threads = std::max((unsigned int) 1, std::thread::hardware_concurrency()); - unsigned int real_num_threads = std::min(hardware_threads, requested_num_threads); + const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); SimpleLogger().Write() << "Input file: " << input_path.filename().string(); SimpleLogger().Write() << "Restrictions file: " << restrictions_path.filename().string(); SimpleLogger().Write() << "Profile: " << profile_path.filename().string(); - SimpleLogger().Write() << "Threads: " << real_num_threads << " (requested " - << requested_num_threads << ")"; + SimpleLogger().Write() << "Threads: " << requested_num_threads; + if (recommended_num_threads != requested_num_threads) + { + SimpleLogger().Write(logWARNING) << "The recommended number of threads is " + << recommended_num_threads + << "! This setting may have performance side-effects."; + } - tbb::task_scheduler_init init(real_num_threads); + tbb::task_scheduler_init init(requested_num_threads); LogPolicy::GetInstance().Unmute(); boost::filesystem::ifstream restriction_stream(restrictions_path, std::ios::binary);