From 12b2242ad56878810664b76ac9044b5b43ca24f1 Mon Sep 17 00:00:00 2001 From: Michael Krasnyk Date: Thu, 21 Sep 2017 16:05:22 +0200 Subject: [PATCH] Add osmium locations cache --- features/options/extract/lua.feature | 45 +++++++++++++-------- src/extractor/extractor.cpp | 29 +++++++++++-- src/extractor/scripting_environment_lua.cpp | 4 +- 3 files changed, 57 insertions(+), 21 deletions(-) diff --git a/features/options/extract/lua.feature b/features/options/extract/lua.feature index 6d0b07f49..0a0c8d11f 100644 --- a/features/options/extract/lua.feature +++ b/features/options/extract/lua.feature @@ -32,22 +32,6 @@ Feature: osrm-extract lua ways:get_nodes() And stdout should contain "node id 2" - Scenario: osrm-extract location-dependent data without add-locations-to-ways preprocessing - Given the profile "testbot" - And the node map - """ - a b - """ - And the ways - | nodes | - | ab | - And the data has been saved to disk - - When I try to run "osrm-extract --profile {profile_file} {osm_file} --location-dependent-data test/data/regions/null-island.geojson" - Then it should exit with an error - And stderr should contain "invalid location" - - Scenario: osrm-extract location-dependent data Given the profile file """ @@ -111,3 +95,32 @@ Feature: osrm-extract lua ways:get_nodes() And stdout should contain "2 GeoJSON polygons" And stdout should contain "ISO3166-1 HK" And stdout should contain "ISO3166-1 none" + + Scenario: osrm-extract location-dependent data via locations cache + Given the profile file + """ + functions = require('testbot') + + function way_function(profile, way, result, relations, location_data) + assert(location_data) + for k, v in pairs(location_data) do print (k .. ' ' .. tostring(v)) end + result.forward_mode = mode.driving + result.forward_speed = 1 + end + + functions.process_way = way_function + return functions + """ + And the node map + """ + a b + """ + And the ways + | nodes | + | ab | + And the data has been saved to disk + + When I run "osrm-extract --profile {profile_file} {osm_file} --location-dependent-data test/data/regions/null-island.geojson" + Then it should exit successfully + And stdout should contain "answer 42" + And stdout should not contain "array" diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index cd3080f1e..3d7e2aabb 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -40,8 +40,11 @@ #include #include +#include +#include #include #include +#include #include #include @@ -341,7 +344,7 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, config.parse_conditionals, restrictions); - using SharedBuffer = std::shared_ptr; + using SharedBuffer = std::shared_ptr; struct ParsedBuffer { SharedBuffer buffer; @@ -351,11 +354,12 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, std::vector resulting_restrictions; }; + // Read OSM data tbb::filter_t buffer_reader( tbb::filter::serial_in_order, [&](tbb::flow_control &fc) { if (auto buffer = reader->read()) { - return std::make_shared(std::move(buffer)); + return std::make_shared(std::move(buffer)); } else { @@ -363,6 +367,22 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, return SharedBuffer{}; } }); + + // Cache node locations (assumes nodes are placed before ways) + using osmium_index_type = + osmium::index::map::SparseMemMap; + using osmium_location_handler_type = osmium::handler::NodeLocationsForWays; + + osmium_index_type location_cache; + osmium_location_handler_type location_handler(location_cache); + + tbb::filter_t location_cacher( + tbb::filter::serial_in_order, [&location_handler](SharedBuffer buffer) { + osmium::apply(buffer->begin(), buffer->end(), location_handler); + return buffer; + }); + + // Process OSM elements tbb::filter_t> buffer_transform( tbb::filter::parallel, [&](const SharedBuffer buffer) { if (!buffer) @@ -379,6 +399,8 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, parsed_buffer->resulting_restrictions); return parsed_buffer; }); + + // tbb::filter_t, void> buffer_storage( tbb::filter::serial_in_order, [&](const std::shared_ptr parsed_buffer) { if (!parsed_buffer) @@ -436,8 +458,9 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment, (config.use_metadata ? osmium::io::read_meta::yes : osmium::io::read_meta::no))); // Number of pipeline tokens that yielded the best speedup was about 1.5 * num_cores + // TODO: make location_cacher conditional tbb::parallel_pipeline(tbb::task_scheduler_init::default_num_threads() * 1.5, - buffer_reader & buffer_transform & buffer_storage); + buffer_reader & location_cacher & buffer_transform & buffer_storage); TIMER_STOP(parsing); util::Log() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds"; diff --git a/src/extractor/scripting_environment_lua.cpp b/src/extractor/scripting_environment_lua.cpp index ce38b9e9d..775e459fe 100644 --- a/src/extractor/scripting_environment_lua.cpp +++ b/src/extractor/scripting_environment_lua.cpp @@ -978,7 +978,6 @@ void LuaScriptingContext::ProcessNode(const osmium::Node &node, } } - namespace { // boost::variant visitor that inserts a key-value pair in a Lua table @@ -1018,7 +1017,8 @@ void LuaScriptingContext::ProcessWay(const osmium::Way &way, } else { - way_function(profile_table, way, result, relations, toLua(state, location_dependent_data(way))); + way_function( + profile_table, way, result, relations, toLua(state, location_dependent_data(way))); } break; case 2: