Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add osmium locations cache

Browse Source
This commit is contained in:
Michael Krasnyk 2017-09-21 16:05:22 +02:00
parent fec2b602a2
commit 12b2242ad5
3 changed files with 57 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
features/options/extract/lua.feature
View File

@ -32,22 +32,6 @@ Feature: osrm-extract lua ways:get_nodes()
And stdout should contain "node id 2"
Scenario: osrm-extract location-dependent data without add-locations-to-ways preprocessing
Given the profile "testbot"
And the node map
"""
a b
"""
And the ways
| nodes |
| ab |
And the data has been saved to disk
When I try to run "osrm-extract --profile {profile_file} {osm_file} --location-dependent-data test/data/regions/null-island.geojson"
Then it should exit with an error
And stderr should contain "invalid location"
Scenario: osrm-extract location-dependent data
Given the profile file
"""
@ -111,3 +95,32 @@ Feature: osrm-extract lua ways:get_nodes()
And stdout should contain "2 GeoJSON polygons"
And stdout should contain "ISO3166-1 HK"
And stdout should contain "ISO3166-1 none"
Scenario: osrm-extract location-dependent data via locations cache
Given the profile file
"""
functions = require('testbot')
function way_function(profile, way, result, relations, location_data)
assert(location_data)
for k, v in pairs(location_data) do print (k .. ' ' .. tostring(v)) end
result.forward_mode = mode.driving
result.forward_speed = 1
end
functions.process_way = way_function
return functions
"""
And the node map
"""
a b
"""
And the ways
| nodes |
| ab |
And the data has been saved to disk
When I run "osrm-extract --profile {profile_file} {osm_file} --location-dependent-data test/data/regions/null-island.geojson"
Then it should exit successfully
And stdout should contain "answer 42"
And stdout should not contain "array"

29
src/extractor/extractor.cpp
View File

@ -40,8 +40,11 @@
#include <boost/optional/optional.hpp>
#include <boost/scope_exit.hpp>
#include <osmium/handler/node_locations_for_ways.hpp>
#include <osmium/index/map/sparse_mem_map.hpp>
#include <osmium/io/any_input.hpp>
#include <osmium/thread/pool.hpp>
#include <osmium/visitor.hpp>
#include <tbb/pipeline.h>
#include <tbb/task_scheduler_init.h>
@ -341,7 +344,7 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
config.parse_conditionals,
restrictions);
using SharedBuffer = std::shared_ptr<const osmium::memory::Buffer>;
using SharedBuffer = std::shared_ptr<osmium::memory::Buffer>;
struct ParsedBuffer
{
SharedBuffer buffer;
@ -351,11 +354,12 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
std::vector<InputConditionalTurnRestriction> resulting_restrictions;
};
// Read OSM data
tbb::filter_t<void, SharedBuffer> buffer_reader(
tbb::filter::serial_in_order, [&](tbb::flow_control &fc) {
if (auto buffer = reader->read())
{
return std::make_shared<const osmium::memory::Buffer>(std::move(buffer));
return std::make_shared<osmium::memory::Buffer>(std::move(buffer));
}
else
{
@ -363,6 +367,22 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
return SharedBuffer{};
}
});
// Cache node locations (assumes nodes are placed before ways)
using osmium_index_type =
osmium::index::map::SparseMemMap<osmium::unsigned_object_id_type, osmium::Location>;
using osmium_location_handler_type = osmium::handler::NodeLocationsForWays<osmium_index_type>;
osmium_index_type location_cache;
osmium_location_handler_type location_handler(location_cache);
tbb::filter_t<SharedBuffer, SharedBuffer> location_cacher(
tbb::filter::serial_in_order, [&location_handler](SharedBuffer buffer) {
osmium::apply(buffer->begin(), buffer->end(), location_handler);
return buffer;
});
// Process OSM elements
tbb::filter_t<SharedBuffer, std::shared_ptr<ParsedBuffer>> buffer_transform(
tbb::filter::parallel, [&](const SharedBuffer buffer) {
if (!buffer)
@ -379,6 +399,8 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
parsed_buffer->resulting_restrictions);
return parsed_buffer;
});
//
tbb::filter_t<std::shared_ptr<ParsedBuffer>, void> buffer_storage(
tbb::filter::serial_in_order, [&](const std::shared_ptr<ParsedBuffer> parsed_buffer) {
if (!parsed_buffer)
@ -436,8 +458,9 @@ Extractor::ParseOSMData(ScriptingEnvironment &scripting_environment,
(config.use_metadata ? osmium::io::read_meta::yes : osmium::io::read_meta::no)));
// Number of pipeline tokens that yielded the best speedup was about 1.5 * num_cores
// TODO: make location_cacher conditional
tbb::parallel_pipeline(tbb::task_scheduler_init::default_num_threads() * 1.5,
buffer_reader & buffer_transform & buffer_storage);
buffer_reader & location_cacher & buffer_transform & buffer_storage);
TIMER_STOP(parsing);
util::Log() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds";

4
src/extractor/scripting_environment_lua.cpp
View File

@ -978,7 +978,6 @@ void LuaScriptingContext::ProcessNode(const osmium::Node &node,
}
}
namespace
{
// boost::variant visitor that inserts a key-value pair in a Lua table
@ -1018,7 +1017,8 @@ void LuaScriptingContext::ProcessWay(const osmium::Way &way,
}
else
{
way_function(profile_table, way, result, relations, toLua(state, location_dependent_data(way)));
way_function(
profile_table, way, result, relations, toLua(state, location_dependent_data(way)));
}
break;
case 2: